Allegro.cc - Online Community

Allegro.cc Forums » Programming Questions » Packfile, Datafile, Compression isn't awesome

Credits go to dudaskank, Fladimir da Gorf, and umperio for helping out!
This thread is locked; no one can reply to it. rss feed Print
Packfile, Datafile, Compression isn't awesome
GullRaDriel
Member #3,861
September 2003
avatar

Heya all. I'm currently trying to get a compressed version of DATAFILE , ala grabber, but it seems that the compression isn't relevant with the way I use, the grabber beat me.

Here is a little test I do for testing if loading is OK, playing, saving.
No crash, but a huge file as big as the whole wav before "compressing"

#SelectExpand
1/**\file test.c 2* 3* Main Nilorea Library Test File 4* 5*\author Castagnier Mickaël 6* 7*\version 1.0 8* 9*\date 22/02/2006 10* 11*/ 12 13 14#include <nilorea.h> 15 16 17int IT = 0, a = 0 , b = 0 ; 18 19PACKFILE *pfile; 20 21LIBSOUND *lib_s; 22 23SAMPLE *SPL; 24 25char cmd[ 512 ]; 26 27struct al_ffblk fi; 28 29 30 31int main( int argc , char *argv[] ) 32 { 33 34 35 allegro_init(); 36 install_timer(); 37 install_mouse(); 38 install_keyboard(); 39 install_sound( DIGI_AUTODETECT , MIDI_NONE , NULL ); 40 41 42 gfx_mode( GFX_DIRECTX_WIN , 800 , 600 , 43 0 , 0 , 44 32 , 45 0 , 0 , 0 , 255 ); 46 47 if ( create_libsound( &lib_s , "nilorea sound library test" , 500 ) == FALSE ) 48 allegro_message( "Creating libsound crashed" ) ; 49 50 if ( !lib_s ) 51 return FALSE; 52 53 54 IT = al_findfirst( "SOUND\*.wav" , &fi , FA_ARCH | FA_RDONLY ); 55 56 /* we loop until we got no result */ 57 while ( IT == 0 ) { 58 59 sprintf( cmd , "SOUND\\%s" , fi.name ); 60 61 clear( screen ); 62 63 SPL = load_sample( cmd ); 64 65 if ( SPL ) { 66 textprintf_ex( screen , font , 10 , 10 , 1 , makecol( 255, 255, 255 ) , "%s loading" , cmd ); 67 a = add_sample_to_lib( &lib_s , SPL , fi.name ); 68 if ( a == -1 ) 69 textprintf_ex( screen , font , 10 , 20 , 1 , makecol( 255, 255, 255 ) , "error adding %s" , cmd ); 70 else { 71 textprintf_ex( screen , font , 10 , 40 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d MAX:%d %s STATE:%d " , lib_s -> name , lib_s -> current , lib_s -> max , lib_s -> data[ a ] . name , lib_s -> data[ a ].state ); 72 } 73 } 74 else 75 textprintf_ex( screen , font , 10 , 30 , 1 , makecol( 255, 255, 255 ) , "error reading %s" , cmd ); 76 77 78 IT = al_findnext( &fi ); 79 80 } 81 82 83 84 al_findclose( &fi ); 85 86 IT = 0; 87 88 a = 0; 89 90 91 play_sample( lib_s -> data[ IT ] . sample , a, 128, 1000, 0 ); 92 93 do { 94 95 if ( key[ KEY_LEFT ] && IT > 0 ) { 96 clear( screen ); 97 stop_sample( lib_s -> data[ IT ].sample ); 98 IT--; 99 play_sample( lib_s -> data[ IT ] . sample , a, 128, 1000, 0 ); 100 textprintf_ex( screen , font , 10 , 40 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d IT:%d %s STATE:%d " , lib_s -> name , lib_s -> current , IT, lib_s -> data[ IT ] . name , lib_s -> data[ IT ].state ); 101 b = search_sample_in( lib_s , lib_s -> data[ IT ] .name ); 102 if ( b != -1 ) 103 textprintf_ex( screen , font , 10 , 50 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d b:%d %s STATE:%d " , lib_s -> name , lib_s -> current , b, lib_s -> data[ b ] . name , lib_s -> data[ b ].state ); 104 105 106 do {} 107 while ( key[ KEY_LEFT ] ); 108 } 109 if ( key[ KEY_RIGHT ] && IT < lib_s -> current - 1 ) { 110 clear( screen ); 111 stop_sample( lib_s -> data[ IT ].sample ); 112 IT++; 113 play_sample( lib_s -> data[ IT ] . sample , a, 128, 1000, 0 ); 114 textprintf_ex( screen , font , 10 , 40 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d IT:%d %s STATE:%d " , lib_s -> name , lib_s -> current , IT , lib_s -> data[ IT ] . name , lib_s -> data[ IT ].state ); 115 b = search_sample_in( lib_s , lib_s -> data[ IT ] .name ); 116 if ( b != -1 ) 117 textprintf_ex( screen , font , 10 , 50 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d b:%d %s STATE:%d " , lib_s -> name , lib_s -> current , b, lib_s -> data[ b ] . name , lib_s -> data[ b ].state ); 118 do {} 119 while ( key[ KEY_RIGHT ] ); 120 } 121 122 if ( key[ KEY_UP ] && a < 255 ) { 123 a += 5; 124 adjust_sample( lib_s -> data[ IT ] .sample, a, 128, 1000, 0 ); 125 do {} 126 while ( key[ KEY_UP ] ); 127 } 128 if ( key[ KEY_DOWN ] && a > 0 ) { 129 a -= 5; 130 adjust_sample( lib_s -> data[ IT ] .sample, a, 128, 1000, 0 ); 131 do {} 132 while ( key[ KEY_UP ] ); 133 } 134 } 135 while ( !key[ KEY_ESC ] ); 136 137 138 pfile = pack_fopen( "sounds.dat" , "wp" ); 139 save_libsound( lib_s , pfile ); 140 pack_fclose( pfile ); 141 142 143 destroy_libsound( &lib_s ); 144 145 return 1; 146 147 } 148END_OF_MAIN();

Here are the functions for saving:

#SelectExpand
1/*!\fn save_sample_in_datafile( SAMPLE *spl , PACKFILE *f ) 2 * 3 *\brief Save a loaded sample in the specified opened datafile 4 * 5 *\param spl The SAMPLE *sample to save 6 *\param f An Opened PACKFILE *file 7 * 8 *\return TRUE or FALSE 9 */ 10 11int save_sample_in_datafile( SAMPLE *spl , PACKFILE *f ) 12{ 13 *allegro_errno = 0; 14 15 pack_mputw((spl->stereo) ? -spl->bits : spl->bits, f); 16 pack_mputw(spl->freq, f); 17 pack_mputl(spl->len, f); 18 19 if (spl->bits == 8) { 20 pack_fwrite(spl->data, spl->len * ((spl->stereo) ? 2 : 1), f); 21 } 22 else { 23 int i; 24 25 for (i=0; i < (int)spl->len * ((spl->stereo) ? 2 : 1); i++) { 26 pack_iputw(((int16_t *)spl->data)<i>, f); 27 } 28 } 29 30 if (*allegro_errno) 31 return FALSE; 32 else 33 return TRUE; 34 } /* save_sample_in_datafile( ... ) */ 35 36 37 38/*!\fn save_libsound( LIBSOUND *lib , PACKFILE *file ) 39 * 40 *\brief Save a sound library in an opened packfile 41 * 42 *\param lib The library to save 43 *\param file The opened packfile where to save the library 44 * 45 *\return TRUE of FALSE 46 */ 47 48int save_libsound( LIBSOUND *lib , PACKFILE *file ) { 49 50 int it = 0, len = 0; 51 52 if( !lib || !file) 53 return FALSE; 54 55 if( !lib -> name ){ 56 Malloc( lib -> name , char , uconvert_size( "defaut" , U_CURRENT, U_UNICODE) ); 57 ustrncpy( lib -> name , "defaut" , ustrlen( "defaut" ) ); 58 } 59 60 pack_iputl( lib -> current , file ); 61 pack_iputl( lib -> max , file ); 62 len = uconvert_size( lib -> name , U_CURRENT, U_UNICODE) 63 pack_iputl( len , file ); 64 pack_fwrite( lib -> name , len , file ); 65 66 for( it = 0 ; it < lib -> current ; it ++ ) { 67 if( lib -> data[ it ] . state != NONE ){ 68 pack_iputl( 1 , file ); 69 pack_iputl( uconvert_size( lib -> data[ it ] . name , U_CURRENT, U_UNICODE) , file ); 70 pack_fwrite( lib -> data[ it ] . name , uconvert_size( lib -> data[ it ] . name , U_CURRENT, U_UNICODE) , file ); 71 save_sample_in_datafile( lib -> data[ it ] . sample , file); 72 } else 73 pack_iputl( 0 , file ); 74 75 } 76 77 return TRUE; 78 79 80} /* save_libsound( ... ) */

With this I got 129 Meg of wav files compressed into ... 145 Megs compressed datafile !

I must be wrong somewhere ...

If anyone have any idea of WTF :P

EDIT: Humpf I'm dumb. I'm just reding somethinh as create_lzss_pack_data in the manual... Am I on the right way ? ...

"Code is like shit - it only smells if it is not yours"
Allegro Wiki, full of examples and articles !!

Fladimir da Gorf
Member #1,565
October 2001
avatar

Of course you could always use .zip files, for example. Allegro uses some crappy compression routines because they don't require additional libraries (like gz).

OpenLayer has reached a random SVN version number ;) | Online manual | Installation video!| MSVC projects now possible with cmake | Now alvailable as a Dev-C++ Devpack! (Thanks to Kotori)

GullRaDriel
Member #3,861
September 2003
avatar

I am taking a look at zlib. My problem will be to manage the dynamic loading-freeing of songs into memory.

Maybe I will add a special flag into 'SOUND' and some test to see if i should load it, for how long, or if it is a 'permanent' sample.

Making ressources handler is a pain in the ass, but it's a necessary war !

"Code is like shit - it only smells if it is not yours"
Allegro Wiki, full of examples and articles !!

dudaskank
Member #561
July 2000
avatar

Hi

I don't have that much experience in allegro packfiles, but if you try to compress some data that is already compressed, like mp3, xvid frames, jpeg, you get a bigger file. But I don't think that is the problem in this situation...

And, for a better compression ratio (better than zip) you can try the LZMA SDK. But the encoder is in C++...

Toque a balada do amor inabalável, eterna love song de nós dois
Eduardo "Dudaskank"
[ Home Page (ptbr) | Blog (ptbr) | Tetris 1.1 (ptbr) | Resta Um (ptbr) | MJpgAlleg 2.3 ]

GullRaDriel
Member #3,861
September 2003
avatar

I'll give it a try, even if I am better at C ;-p

"Code is like shit - it only smells if it is not yours"
Allegro Wiki, full of examples and articles !!

umperio
Member #3,474
April 2003
avatar

Maybe it is not exactly what you want but you could look into physfs which is a nice resource handler.

http://icculus.org/physfs/

Regards

GullRaDriel
Member #3,861
September 2003
avatar

Umpf ! umperio...
That a great link !
It's free, downloadable, api, doc, binaries, src ! And even a tutorial.

Now the thread is OK :-p

"Code is like shit - it only smells if it is not yours"
Allegro Wiki, full of examples and articles !!

Arthur Kalliokoski
Second in Command
February 2005
avatar

Sound that's writtend directly to a file rarely compresses well, because the data isn't very redundant. If you encode the deltas between successive samples and compress those, it works much better. In other words, the sloping "side" of a sound wave is relatively constant. There's sound formats that already do this but I forget the name. PCM?

“Throughout history, poverty is the normal condition of man. Advances which permit this norm to be exceeded — here and there, now and then — are the work of an extremely small minority, frequently despised, often condemned, and almost always opposed by all right-thinking people. Whenever this tiny minority is kept from creating, or (as sometimes happens) is driven out of a society, the people then slip back into abject poverty. This is known as "bad luck.”

― Robert A. Heinlein

Kitty Cat
Member #2,815
October 2002
avatar

ADPCM, Adaptive Differential Pulse Code Modulation. PCM is the standard, uncompressed data.

--
"Do not meddle in the affairs of cats, for they are subtle and will pee on your computer." -- Bruce Graham

Richard Phipps
Member #1,632
November 2001
avatar

If you have 129 meg of wav's than I'd look at using OGG files instead.

Evert
Member #794
November 2000
avatar

If you're interested in reading ZIP files, then there's an addon that allows you to do that. See http://www.allegro.cc/forums/thread/467091

Tobias Dammers
Member #2,604
August 2002
avatar

For large amounts of sound data, I'd strongly suggest you use some sort of lossy compression (ogg is good, mp3 has legal issues, wma even more so). Unless you do heavy processing on the audio, or use insane compression settings, you should be fine.
For music, you might also want to try tracker music instead of actual audio files; because of the typical repetitiveness of computer game music, tracker files are usually much smaller than audio files.
Finally, consider lowering audio quality. You don't need 96kHz, 24bit 5.1 samples. For a game, most sound effects can be 22.5 kHz, 16 bit, mono (some profit from being stereo though), without making much of an audible difference.

If all else fails, distribute on CD/DVD only :P

---
Me make music: Triofobie
---
"We need Tobias and his awesome trombone, too." - Johan Halmén

GullRaDriel
Member #3,861
September 2003
avatar

Richard said:

I'd look at using OGG files instead

My bad, I never achieve to make any ogg playing with alogg or dumb+ogg

Evert said:

If you're interested in reading ZIP files

Not for the same thing as the topic, but I take the link ;-)

Tobias said:

If all else fails, distribute on CD/DVD only

Since all the game/source/stuff is spread around the world by the magic of internet, it's not possible. And I don't have a kopeck to buy and send some copy.

"Code is like shit - it only smells if it is not yours"
Allegro Wiki, full of examples and articles !!

HoHo
Member #4,534
April 2004
avatar

You could always provide two versions:
One with small size and compressed audio
Other with big size and uncompressed audio.

Perhaps you can even charge a few $ for the uncompressed thingie :)

__________
In theory, there is no difference between theory and practice. But, in practice, there is - Jan L.A. van de Snepscheut
MMORPG's...Many Men Online Role Playing Girls - Radagar
"Is Java REALLY slower? Does STL really bloat your exes? Find out with your friendly host, HoHo, and his benchmarking machine!" - Jakub Wasilewski

Ron Novy
Member #6,982
March 2006
avatar

WavePack is a good lossless audio compression format but its very slow. It might be good for distributing large wav audio files that you don't want to compress into a lossy format. Just a note: the PACKFILE compresion uses an LZW style compresion that is not audio friendly. Audio data is to complex for LZW style compresion. Maybe its time to create a media friendly compression format that selects the compression based on the contents. Maybe a plug-in style compressor.

?Light bulb :)

typedef struct C_VTABLE {
   char *name;     /* name of plugin */
   char *author;   /* Name of author + email address + websit.... */
   char *file_ext; /* input extensions that might be relavent to this plug */
   int  type;      /* Type of data this plug in understands: TEXT=0, BIN=1, 
                    * BITMAP=2, AUDIO=3, VIDEO=4... 
                    */
   int  init_compressor(DC, format_info, ...);
   void *compress(DC, void *src, int size_in, int *size_out);
   int  init_decompressor(DC, format_info, ...);
   void *decompress(DC, void *src, int size_in, int *size_out);
} C_VTABLE;

----
Oh... Bieber! I thought everyone was chanting Beaver... Now it doesn't make any sense at all. :-/

GullRaDriel
Member #3,861
September 2003
avatar

Ron Novy, that's a nice start. I'm still waiting for the plug ;D

"Code is like shit - it only smells if it is not yours"
Allegro Wiki, full of examples and articles !!

Go to: