You could use a transparent png or svg to create the shape overlay, with the blank areas in effect becoming the mask or you could use the clip or mask property in CSS.
Here's a jsFiddle example using svg.
External JS/CSS:
<link href="http://vjs.zencdn.net/4.3/video-js.css" rel="stylesheet">
<script src="http://vjs.zencdn.net/4.3/video.js"></script>
HTML:
<body>
<!--VideoJS linked to default video for placeholder -->
<div id="videoDiv">
<video id="example_video_1" class="video-js vjs-default-skin" autoplay controls preload="none" width="900" height="600"
poster="http://video-js.zencoder.com/oceans-clip.png"
data-setup="{}">
<source src="http://video-js.zencoder.com/oceans-clip.mp4" type='video/mp4' />
<source src="http://video-js.zencoder.com/oceans-clip.webm" type='video/webm' />
<source src="http://video-js.zencoder.com/oceans-clip.ogv" type='video/ogg' />
<track kind="captions" src="demo.captions.vtt" srclang="en" label="English"></track><!-- Tracks need an ending tag thanks to IE9 -->
<track kind="subtitles" src="demo.captions.vtt" srclang="en" label="English"></track><!-- Tracks need an ending tag thanks to IE9 -->
</video>
</div>
<!--End Video JS-->
<!--SVG Mask, output from Adobe Illustrator -->
<div id="wrapper">
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="900px" height="600px" viewBox="0 0 900 600" enable-background="new 0 0 900 600" xml:space="preserve">
<g>
<path stroke="#FFFFFF" stroke-miterlimit="10" d="M0,0v600h900V0H0z M316.549,351.231h-30.796v-92.803l-23.729,92.803h-27.875
l-23.685-92.803v92.803h-30.796V229.542h49.452l19.07,74.043l18.936-74.043h49.422V351.231z M421.439,351.231l-6.088-20.088h-42.69
l-5.932,20.088h-38.394l45.737-121.689h41.017l45.727,121.689H421.439z M570.138,333.218c-4.316,6.586-10.349,11.58-18.096,14.983
c-7.748,3.403-17.515,5.105-29.302,5.105c-20.697,0-35.029-3.984-42.998-11.953s-12.479-18.096-13.53-30.381l35.776-2.241
c0.774,5.811,2.352,10.238,4.731,13.281c3.873,4.926,9.407,7.388,16.602,7.388c5.367,0,9.505-1.258,12.409-3.777
c2.906-2.517,4.358-5.437,4.358-8.757c0-3.154-1.384-5.977-4.15-8.467c-2.768-2.49-9.187-4.842-19.258-7.056
c-16.491-3.707-28.251-8.633-35.278-14.775c-7.084-6.143-10.625-13.973-10.625-23.491c0-6.253,1.812-12.161,5.438-17.722
c3.623-5.562,9.074-9.932,16.352-13.115c7.276-3.182,17.252-4.773,29.925-4.773c15.55,0,27.405,2.892,35.569,8.674
c8.161,5.783,13.018,14.983,14.567,27.6l-35.444,2.075c-0.941-5.479-2.92-9.463-5.936-11.953c-3.017-2.49-7.18-3.735-12.492-3.735
c-4.372,0-7.665,0.927-9.878,2.781c-2.214,1.855-3.32,4.109-3.32,6.765c0,1.938,0.913,3.681,2.739,5.229
c1.771,1.606,5.977,3.1,12.617,4.482c16.436,3.542,28.208,7.126,35.32,10.75c7.109,3.625,12.285,8.122,15.521,13.489
c3.238,5.368,4.856,11.372,4.856,18.013C576.612,319.439,574.454,326.633,570.138,333.218z M678.392,351.231l-25.657-50.045
l-19.428,20.35v29.695h-37.603V229.542h37.603v45.986l39.389-45.986h50.011L678.3,275.473l46.398,75.758H678.392z"/>
<polygon stroke="#FFFFFF" stroke-miterlimit="10" points="380.828,304.83 407.552,304.83 394.12,261.084 "/>
</g>
</svg>
</div>
<!--End SVG Mask-->
</body>
CSS:
#wrapper {
position: absolute;
height: 100%;
width: 100%;
z-index: 5;
}
#videoDiv {
position: absolute;
z-index: -5;
}
