FreeWRL / FreeX3D 4.3.0
MPEG_Utils.c
1/*
2
3
4???
5
6*/
7
8/****************************************************************************
9 This file is part of the FreeWRL/FreeX3D Distribution.
10
11 Copyright 2009 CRC Canada. (http://www.crc.gc.ca)
12
13 FreeWRL/FreeX3D is free software: you can redistribute it and/or modify
14 it under the terms of the GNU Lesser Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
17
18 FreeWRL/FreeX3D is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
22
23 You should have received a copy of the GNU General Public License
24 along with FreeWRL/FreeX3D. If not, see <http://www.gnu.org/licenses/>.
25****************************************************************************/
26
27
28/* NOTE: we have to re-implement the loading of movie textures; the code in here was a decade old and did not
29keep up with "the times". Check for ifdef HAVE_TO_REIMPLEMENT_MOVIETEXTURES in the code */
30
31/* July 2016 note:
32 http://www.web3d.org/documents/specifications/19775-1/V3.3/Part01/components/texturing.html#MovieTexture
33 - specs say to support MPEG1
34 - online says all patents have expired for MPEG1 and its Level II audio
35 http://www.web3d.org/x3d/content/examples/ConformanceNist/
36 - see Movie texture example with VTS.mpg
37
38 A. mpeg1 capable library plumbing
39 There are still patents and licensing issues with recent mp4 and even mpeg-2 I've heard.
40 MPEG1 - any patents have expired
41 Goal: a generic interface we can use to wrap 3 implementations:
42 1. stub
43 2. option: old-fashioned cross-platform mpeg1 c code, as fallback/default
44 3. platform-supplied/platform-specific video API/libraries
45 and to use both audio and video streams, with audio compatible with SoundSource node api needs.
46
47 Links >
48 2. old fashioned mpeg1
49 Mpeg-1 libs / code
50 2.1 reference implementation
51 https://en.wikipedia.org/wiki/MPEG-1
52 See source code link at bottom, reference implementation
53 which has an ISO license which dug9 reads to mean:
54 - use for simulation of what electronic devices will do, presumably by electronics companies
55 - (but not intendended / foreseen as a software library product, uncertain if allowed)
56 it includes audio and video
57 2.2 Berkley mpeg_play derivitives
58 a) berkley_brown
59 original freewrl implementation (taken out in 2009), has both berkley and brown U listed in license section
60 b) berkley_gerg
61 http://www.gerg.ca/software/mpeglib/
62 Mpeg1 - hack of berkley mpeg1 code, no separate license on hacks show
63 c) berkley_nabg
64 http://www.uow.edu.au/~nabg/MPEG/mpeg1.html
65 Mpeg1 explanation with hacked Berkley code and license: berkley + do-what-you-want-with-hacked-code
66 2.3 ffmpeg.org
67 LGPL by default (can add in GPL parts, we don't need)
68 but may include patented algorithms for post-MPEG1-Audio_Level_II
69 but has a way to limit codecs available at runtime:
70 instead of load_all() you would somehow say which ones to load?? see dranger tutorial about tut5
71 and if so and you limit codecs to MPEG1 with Level II audio, then
72 ffplay.c and tutorial http://dranger.com/ffmpeg/ can help,
73 substituting freewrl's audio lib's API, pthreads, and our openGL plumbing for SDL
74 might be able to #define some pthread for SDL thread functions in ffplay.c
75 - except don't assume freewrl's pthreads is complete: no cancel, its an emulated lib on some platforms
76 https://github.com/FFmpeg/FFmpeg
77 https://github.com/FFmpeg/FFmpeg/blob/master/ffplay.c
78 line interesting function
79 3352 event_loop() - play, pause, rewind user functions
80 3000 infinite_buffer real_time -in theory could pull from url instead of file
81 2660 forcing codec by name - if blocking patented MPEG2+ / sticking to MPEG1+LevelII audio
82 2400 audio_decode_frame - somwhere we need to get the audio PCM decompressed buffer, so we can pass to our audio API
83 1808 get_video_frame - somewhere we need to get the current frame, perhaps from do_MovieTextureTick() we would get closest-to-current frame and freeze it for opengl
84 1506 video_refresh called to display each frame
85
86
87 3. Platform supplied
88 3.1 windows > directX audio and video (I have the dx audio working in one app)
89 3.2 android >
90 3.3 linux desktop >
91
92 B. freewrl texture and sound plumbing
93 MAJOR DESIGN OPTIONS:
94 I. Process video frame into mipmapped opengl texture:
95 a) on loading:, pre-process entire video into mipmapped opengl textures, one per video frame (2009 implementation)
96 Disadvantage: 30fps x 10 seconds is 300 textures - a lot to store, and prepare if unneeded
97 x a lot of opengl textures needed - exhaustion/limits?
98 Benefit: faster per frame when playing
99 b) on-the-fly during play: in a separate thread, process video frames to replace single opengl texture
100 Benefit: vs c: mpeg decompression: successive frames require previous frame decoded, so state is managed
101 - vs a,c: thread can do its own throttling
102 - vs a: storage is just the decompression sequence
103 Disadvantage: single core will be doing a lot of un-needed mipmapping
104 x thread collisions - texture being replaced in one thread and drawn in another?
105 x stereo vision > left and right eye might see different texture
106 c) on-the-fly in do_tick (once per render frame), interpolate closest frame based on time, replace single opengl texture
107 Benefit: vs a,b: no unnecessary frames interpolated, no unnecessary mipmapping in any thread, just the frame needed
108 vs. b: same frame appears in left/right stereo, no timing weirdness
109 vs. a: storage is just the decompression sequence
110 d) combo of b) and c): separate thread prepares small set of raw frames,
111 do_MovieTextureTick() picks one or asks for one and opengl-izes it
112 II. Support streaming?
113 a) Continuous Streaming video from url or
114 b) just finite local file
115 SIMPLIFYING DECISION: b) finite local file
116 III. Separate thread for decoding / interpolating frames?
117 a) or in rendering thread (any throttling problems?)
118 b) new separate thread (but what about mipmapping and opengizing interpolated frame?)
119 c) (somehow) use texture thread which is currently parked/unused once images are mipmapped
120 - but currently triggered during image file loading process
121 - could set a flag to an earlier state and re-submit?
122 SIMPLIFYING DECISION: depends on implementation ie what libs you get and how easy it is
123
124 A few facts / details:
125 input media: MPEG1 contains Audio and/or Video
126 Nodes: which we want to use to support SoundSource and/or Texture2D
127 Texture2D: shows one frame at a time in opengl
128 SoundSource: used as / like an AudioClip for Sound node. AudioClip has its own private thread
129 It doesn't make sense to load a movietexture 2x if using for both texture and sound.
130 - you would DEF for one use, and USE for the other
131 - then when you play, you play both USEs at the same time so audio and video are synced
132 Sound is handled per-node.
133 Textures have an intermediary texturetableindexstruct
134
135 2003 - 2009 freewrl movietexture:
136 - on load: decoded and created an opengl texture for each movie frame
137 - used berkley mpeg aka berkley-brown
138
139
140 Proposed freewrl plumbing:
141
142 1. for texture rendering, MovieTexture works like ImageTexture on each render_hier frame,
143 with a single opengl texture number
144 2. leave it to the library-specifics to decide if
145 a) decode-on-load
146 b) decode in a separate thread, anticipatory/queue
147 c) decode-on-demand
148 3. the pause, stop, play, rewind interface is usable as SoundSource like Audioclip, analogous to AudioClip
149 4. perl > make AudioClip and MovieTexture fields in same order, so MovieTexture can be up-caste to AudioClip DONE
150
151 top level interface:
152 X3DMovieTexture._movie; an opaque pointer that will hold a malloced struct representing
153 the movie container and streams; different implementations will be different
154 movie_load() - like loadTextures.c image loaders - takes local path and gets things started
155 puts intial load into ._movie of the requesting node ie res->(wheretoplacedata,offset) = (MovieTexture,_movie)
156 do_MovieTextureTick()
157 in senseInterp.c, once per frame (so stereo uses same movie frame for left/right)
158 could ask for closest frame based on movie time
159 node->tti->texdata = getClosestMovieFrame(movietime)
160 a) decode-on-load would have the frame ready in a list
161 b) multi-thread anticipatory decode would have a private queue/list of decoded frames,
162 and get the closest one, and discard stale frames, and restart decode queue to fill up
163 queue again
164 c) decode-on-demand would decode on demand
165 Texture2D(,,,node->tti->opeglTexture,,,node->tti->texdata) //reset texture data
166
167 loadstatus_MovieTexture(struct X3D_MovieTexture *node) - loadsensor can check if file loaded
168 loadstatus_AudioClip(struct X3D_AudioClip *node) - loadsensor can check if file loaded
169 locateAudioSource (struct X3D_AudioClip *node) - will work for MovieTexture
170
171 search code for MovieTexture, resm_movie to see all hits
172
173
174 MPEG_Utils_berkley.c Nov 15, 2016: compiled but bombs on loading even simple vts.mpg
175 MPEG_Utils_libmpeg2.c Nove 15, 2016: not attempted to implement
176 x undocumented code
177 x GPL
178 x uses callback for frames
179 x no audio channel
180 - but small test program mpeg2dec does run on windows
181 - code seems lite / compact
182*/
183#include <config.h>
184#include <system.h>
185#include <system_threads.h>
186#include <display.h>
187#include <internal.h>
188#include "vrml_parser/CRoutes.h"
189#include "vrml_parser/Structs.h"
190#include "main/ProdCon.h"
191#include "../opengl/OpenGL_Utils.h"
192#include "../opengl/Textures.h"
193#include "../opengl/LoadTextures.h"
194#include "../scenegraph/Component_CubeMapTexturing.h"
195
196#include <list.h>
197#include <io_files.h>
198#include <io_http.h>
199
200#include <threads.h>
201
202#include <libFreeWRL.h>
203
204//put your choice in your config.h (or windows preprocessor directives):
205//#define MOVIETEXTURE_STUB 1 //default
206//#define MOVIETEXTURE_BERKLEYBROWN 1
207//#define MOVIETEXTURE_FFMPEG 1
208//#define MOVIETEXTURE_LIBMPEG2 1
209
210//Option A.
211// movie_load - load as BLOB using standard FILE2BLOB in io_files.c retval = resource_load(res); //FILE2BLOB
212// parse_movie - converts BLOB to sound and video parts, returns parts
213//Option B.
214// movie_load - parse movie as loading
215// parse_movie - return movie parts
216
217#ifdef MOVIETEXTURE_BERKLEYBROWN
218#include "MPEG_Utils_berkley.c"
219#elif MOVIETEXTURE_FFMPEG
220//#include "MPEG_Utils_ffmpeg.c"
221int movie_load_from_file(char *fname, void **opaque);
222double movie_get_duration(void *opaque);
223unsigned char *movie_get_frame_by_fraction(void *opaque, float fraction, int *width, int *height, int *nchan);
224unsigned char * movie_get_audio_PCM_buffer(void *opaque,int *freq, int *channels, int *size, int *bits);
225#include "sounds.h"
226//BufferData * alutBufferDataConstruct (ALvoid *data, size_t length, ALint numChannels,
227// ALint bitsPerSample, ALfloat sampleFrequency);
228
229#define LOAD_STABLE 10 //from Component_Sound.c
230#elif MOVIETEXTURE_LIBMPEG2
231#endif
232
233bool movie_load(resource_item_t *res){
234 bool retval;
235 // see io_files.c for call place
236 //Option A: just load blob for later
237 // retval = resource_load(res); //FILE2BLOB
238 //Option B:
239 // parse during load
240 // copied from imagery_load - but TEX_READ flag will be wrong for movie
241 //int textureNumber;
242 //struct textureTableIndexStruct *entry; // = res->whereToPlaceData;
243 //textureNumber = res->textureNumber;
244 //if(res->status == ress_downloaded){
245 // entry = getTableIndex(textureNumber);
246 // if(entry)
247 // if (movie_load_from_file(entry, res->actual_file)) {
248 // entry->status = TEX_READ; // tell the texture thread to convert data to OpenGL-format
249 // res->status = ress_loaded;
250 // retval = TRUE;
251 // return retval;
252 // }
253 //}
254 //res->status = ress_not_loaded;
255 retval = FALSE;
256
257#ifdef MOVIETEXTURE_STUB
258 res->status = ress_loaded;
259 retval = TRUE;
260#elif MOVIETEXTURE_BERKLEYBROWN
261 {
262 int x,y,depth,frameCount;
263 char *ptr;
264 ptr=NULL;
265 //H: this returns something like a volume image, with slices packed into ptr, and z=frameCount, nchannels = depth.
266 //Q: what's the 'normal' frame rate? should that be returned too, or is there a standard/default?
267 //Nov 15, 2016: bombs on small test file vts.mpg
268 mpg_main(res->actual_file, &x,&y,&depth,&frameCount,&ptr);
269 #ifdef TEXVERBOSE
270 printf ("have x %d y %d depth %d frameCount %d ptr %d\n",x,y,depth,frameCount,ptr);
271 #endif
272 // store_tex_info(loadThisTexture, depth, x, y, ptr,depth==4);
273
274 // and, manually put the frameCount in.
275 //res->frames = frameCount;
276 }
277
278#elif MOVIETEXTURE_FFMPEG
279 {
280 void *opaque;
281 int loaded;
282 loaded = movie_load_from_file(res->actual_file,&opaque);
283 retval = loaded > -1 ? TRUE : FALSE;
284 if(loaded){
285 int freq,channels,size,bits;
286 unsigned char * pcmbuf;
287 struct X3D_MovieTexture *node;
288
289 res->status = ress_loaded;
290 res->complete = TRUE;
291 res->status = ress_parsed; //we'll skip the parse_movie/load_from_blob handler
292
293 node = (struct X3D_MovieTexture *) res->whereToPlaceData;
294 //AUDIO AND/OR VIDEO CHANNELS?
295 node->duration_changed = movie_get_duration(opaque);
296 node->__fw_movie = opaque;
297 node->__loadstatus = LOAD_STABLE;
298 //VIDEO CHANNEL?
299 //double totalframes = node->duration_changed * 30.0;
300 node->speed = 1.0; //1 means normal speed 30.0 / totalframes; //in fractions per second = speed in frames/second / totalframes
301 MARK_EVENT (X3D_NODE(node), offsetof(struct X3D_MovieTexture, duration_changed));
302 //AUDIO CHANNEL?
303 //node->__sourceNumber = parse_movie(node,buffer,len); //__sourceNumber will be openAL buffer number
304 pcmbuf = movie_get_audio_PCM_buffer(opaque,&freq,&channels,&size,&bits);
305 if(pcmbuf){
306 //MPEG1 level1,2 are compressed audio
307 //decoders generally deliver so called PCM pulse code modulated buffers
308 //and that's what audio drivers on computers normally take
309 //and same with the APIs that wrap the hardware drivers ie openAL API
310 printf("audio freq %d channels %d size %d bits per channel %d\n",freq,channels,size,bits);
311 #ifdef HAVE_OPENAL
312 // http://open-activewrl.sourceforge.net/data/OpenAL_PGuide.pdf
313 // page 6
314 {
315 int format;
316 ALuint albuffer;
317 static int once = 0;
318 if(!once){
319 #ifdef HAVE_ALUT
320 //alutInit(0, NULL); // Initialize OpenAL
321 if (!alutInitWithoutContext(NULL, NULL))
322 ConsoleMessage("ALUT init failed\n");
323 #endif //HAVE_ALUT
324 alGetError(); // Clear Error Code
325 //SoundEngineInit();
326 once = 1;
327 }
328
329 alGenBuffers(1, &albuffer);
330 //al.h
331 //#define AL_FORMAT_MONO8 0x1100
332 //#define AL_FORMAT_MONO16 0x1101
333 //#define AL_FORMAT_STEREO8 0x1102
334 //#define AL_FORMAT_STEREO16 0x1103
335 //if(bits == 8)
336 // format = AL_FORMAT_MONO8;
337 //else
338 // format = AL_FORMAT_MONO16;
339 //if(channels == 2)
340 // if(bits == 8)
341 // format = AL_FORMAT_STEREO8;
342 // else
343 // format = AL_FORMAT_STEREO16;
344 format = 0;
345 switch(bits){
346 case 8:
347 format = AL_FORMAT_MONO8;
348 if(channels == 2)
349 format = AL_FORMAT_STEREO8;
350 break;
351 case 16:
352 format = AL_FORMAT_MONO16;
353 if (channels == 2)
354 format = AL_FORMAT_STEREO16;
355 break;
356 case 32:
357 #ifdef AL_EXT_float32
358 format = AL_FORMAT_MONO_FLOAT32;
359 if (channels == 2)
360 format = AL_FORMAT_STEREO_FLOAT32;
361 break;
362 #endif
363 default:
364 break;
365 }
366 if(format > 0){
367 //this is a complex function that tries to figure out if its float, int PCM etc
368 alBufferData(albuffer,format,pcmbuf,size,freq);
369 //BufferData * bdata = _alutBufferDataConstruct( pcmbuf,size,channels,bits, freq);
370
371 node->__sourceNumber = albuffer;
372 }
373 }
374 #endif //HAVE_OPENAL
375 }
376 }
377
378 printf("opqaue = %p, loaded=%d \n",opaque,res->status);
379 }
380#elif MOVIETEXTURE_LIBMPEG2
381#endif
382 return retval;
383}
384int parse_audioclip(struct X3D_AudioClip *node,char *bbuffer, int len);
385int parse_movie(struct X3D_MovieTexture *node, char *buffer,int len){
386 //Option B - parse blob
387 //if your movie api will take a blob, you can call it from here to parse
388 //convert BLOB (binary large object) into video and audio structures
389 //Option A and B - return audio and video parts
390 int audio_sourcenumber;
391 audio_sourcenumber = -1; //BADAUDIOSOURCE
392 //MPEG1 level1,2 are compressed audio
393 //decoders generally deliver so called PCM pulse code modulated buffers
394 //and that's what audio drivers on computers normally take
395 //and same with the APIs that wrap the hardware drivers ie openAL API
396#ifdef MOVIETEXTURE_STUB
397#elif MOVIETEXTURE_BERKLEYBROWN
398#elif MOVIETEXTURE_FFMPEG
399#elif MOVIETEXTURE_LIBMPEG2
400#endif
401 return audio_sourcenumber;
402}
403double compute_duration(int ibuffer);
404
405bool process_res_movie(resource_item_t *res){
406 // METHOD_LOAD_ON_DEMAND
407 //you'll get in here if you didn't (completely) handle movie_load from file
408 //
409 //s_list_t *l;
410 openned_file_t *of;
411 char *buffer;
412 int len;
413 struct X3D_MovieTexture *node;
414
415 buffer = NULL;
416 len = 0;
417 switch (res->type) {
418 case rest_invalid:
419 return FALSE;
420 break;
421
422 case rest_string:
423 buffer = res->URLrequest;
424 break;
425 case rest_url:
426 case rest_file:
427 case rest_multi:
428 of = res->openned_files;
429 if (!of) {
430 /* error */
431 return FALSE;
432 }
433
434 buffer = of->fileData;
435 len = of->fileDataSize;
436 break;
437 }
438
439 node = (struct X3D_MovieTexture *) res->whereToPlaceData;
440 //node->__FILEBLOB = buffer;
441 node->__sourceNumber = parse_movie(node,buffer,len); //__sourceNumber will be openAL buffer number
442 if(node->__sourceNumber > -1) {
443 node->duration_changed = compute_duration(node->__sourceNumber);
444 MARK_EVENT (X3D_NODE(node), offsetof(struct X3D_MovieTexture, duration_changed));
445 return TRUE;
446 }
447 return FALSE;
448}
449
450
451// - still needed ? don't know depends on implementation
452//void getMovieTextureOpenGLFrames(int *highest, int *lowest,int myIndex) {
453// textureTableIndexStruct_s *ti;
454//
456// printf ("getMovieTextureOpenGLFrames, myIndex is ZERL\n");
457// *highest=0; *lowest=0;
458// } else {
459//*/
460// *highest=0; *lowest=0;
461//
462// #ifdef TEXVERBOSE
463// printf ("in getMovieTextureOpenGLFrames, calling getTableIndex\n");
464// #endif
465//
466// ti = getTableIndex(myIndex);
467//
469// if (ti->OpenGLTexture != TEXTURE_INVALID) {
470// *lowest = ti->OpenGLTexture;
471// *highest = 0;
473// }
475//}
476
477unsigned char *movietexture_get_frame_by_fraction(struct X3D_Node* node, float fraction, int *width, int *height, int *nchan){
478 unsigned char* retval = NULL;
479 if(node && node->_nodeType == NODE_MovieTexture){
480 struct X3D_MovieTexture *movietexture = (struct X3D_MovieTexture *)node;
481#ifdef MOVIETEXTURE_STUB
482#elif MOVIETEXTURE_BERKLEYBROWN
483#elif MOVIETEXTURE_FFMPEG
484 retval = movie_get_frame_by_fraction(movietexture->__fw_movie,fraction,width,height,nchan);
485#elif MOVIETEXTURE_LIBMPEG2
486#endif
487 }
488 return retval;
489}