Packfile, Datafile, Compression isn't awesome

GullRaDriel

Heya all. I'm currently trying to get a compressed version of DATAFILE , ala grabber, but it seems that the compression isn't relevant with the way I use, the grabber beat me.

Here is a little test I do for testing if loading is OK, playing, saving.
No crash, but a huge file as big as the whole wav before "compressing"

#SelectExpand
   1/**\file test.c
   2*
   3*  Main Nilorea Library Test File
   4*
   5*\author Castagnier Mickaël
   6*
   7*\version 1.0
   8*
   9*\date 22/02/2006
  10*
  11*/
  12
  13
  14#include <nilorea.h>
  15
  16
  17int IT = 0, a = 0 , b = 0 ;
  18
  19PACKFILE *pfile;
  20
  21LIBSOUND *lib_s;
  22
  23SAMPLE *SPL;
  24
  25char cmd[ 512 ];
  26
  27struct al_ffblk fi;
  28
  29
  30
  31int main( int argc , char *argv[] )
{


allegro_init();
install_timer();
install_mouse();
install_keyboard();
install_sound( DIGI_AUTODETECT , MIDI_NONE , NULL );


gfx_mode( GFX_DIRECTX_WIN , 800 , 600 ,
      0 , 0 ,
      32 ,
      0 , 0 , 0 , 255 );
  
if ( create_libsound( &lib_s , "nilorea sound library test" , 500 ) == FALSE )
  allegro_message( "Creating libsound crashed" ) ;
  
if ( !lib_s )
  return FALSE;
  
  
IT = al_findfirst( "SOUND\*.wav" , &fi , FA_ARCH | FA_RDONLY );

/* we loop until we got no result */
while ( IT == 0 ) {

    sprintf( cmd , "SOUND\\%s" , fi.name );
    
    clear( screen );
    
    SPL = load_sample( cmd );
    
    if ( SPL ) {
        textprintf_ex( screen , font , 10 , 10 , 1 , makecol( 255, 255, 255 ) , "%s loading" , cmd );
        a = add_sample_to_lib( &lib_s , SPL , fi.name );
        if ( a == -1 )
          textprintf_ex( screen , font , 10 , 20 , 1 , makecol( 255, 255, 255 ) , "error adding %s" , cmd );
        else {
            textprintf_ex( screen , font , 10 , 40 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d MAX:%d %s STATE:%d " , lib_s -> name , lib_s -> current , lib_s -> max , lib_s -> data[ a ] . name , lib_s -> data[ a ].state );
            }
        }
    else
      textprintf_ex( screen , font , 10 , 30 , 1 , makecol( 255, 255, 255 ) , "error reading %s" , cmd );
      
      
    IT = al_findnext( &fi );
    
    }
    
    
    
al_findclose( &fi );

IT = 0;

a = 0;


play_sample( lib_s -> data[ IT ] . sample , a, 128, 1000, 0 );

do {

    if ( key[ KEY_LEFT ] && IT > 0 ) {
        clear( screen );
        stop_sample( lib_s -> data[ IT ].sample );
        IT--;
        play_sample( lib_s -> data[ IT ] . sample , a, 128, 1000, 0 );
        textprintf_ex( screen , font , 10 , 40 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d IT:%d %s STATE:%d " , lib_s -> name , lib_s -> current , IT, lib_s -> data[ IT ] . name , lib_s -> data[ IT ].state );
        b = search_sample_in( lib_s , lib_s -> data[ IT ] .name );
        if ( b != -1 )
          textprintf_ex( screen , font , 10 , 50 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d b:%d %s STATE:%d " , lib_s -> name , lib_s -> current , b, lib_s -> data[ b ] . name , lib_s -> data[ b ].state );
          
          
      do {}
        while ( key[ KEY_LEFT ] );
        }
    if ( key[ KEY_RIGHT ] && IT < lib_s -> current - 1 ) {
        clear( screen );
        stop_sample( lib_s -> data[ IT ].sample );
        IT++;
        play_sample( lib_s -> data[ IT ] . sample , a, 128, 1000, 0 );
        textprintf_ex( screen , font , 10 , 40 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d IT:%d %s STATE:%d " , lib_s -> name , lib_s -> current , IT , lib_s -> data[ IT ] . name , lib_s -> data[ IT ].state );
        b = search_sample_in( lib_s , lib_s -> data[ IT ] .name );
        if ( b != -1 )
          textprintf_ex( screen , font , 10 , 50 , 1 , makecol( 255, 255, 255 ) , "%s CUR:%d b:%d %s STATE:%d " , lib_s -> name , lib_s -> current , b, lib_s -> data[ b ] . name , lib_s -> data[ b ].state );
      do {}
        while ( key[ KEY_RIGHT ] );
        }
        
    if ( key[ KEY_UP ] && a < 255 ) {
        a += 5;
        adjust_sample( lib_s -> data[ IT ] .sample, a, 128, 1000, 0 );
        do {}
        while ( key[ KEY_UP ] );
        }
    if ( key[ KEY_DOWN ] && a > 0 ) {
        a -= 5;
        adjust_sample( lib_s -> data[ IT ] .sample, a, 128, 1000, 0 );
        do {}
        while ( key[ KEY_UP ] );
        }
    }
while ( !key[ KEY_ESC ] );


pfile = pack_fopen( "sounds.dat" , "wp" );
save_libsound( lib_s , pfile );
pack_fclose( pfile );


destroy_libsound( &lib_s );

return 1;

}
 148END_OF_MAIN();

Here are the functions for saving:

#SelectExpand
  1/*!\fn save_sample_in_datafile( SAMPLE *spl , PACKFILE *f )
  2 *
  3 *\brief Save a loaded sample in the specified opened datafile
  4 *
  5 *\param spl The SAMPLE *sample to save
  6 *\param f An Opened PACKFILE *file
  7 *
  8 *\return TRUE or FALSE
  9 */
 10
 11int save_sample_in_datafile( SAMPLE *spl , PACKFILE *f )
 12{
 13   *allegro_errno = 0;
 14
 15   pack_mputw((spl->stereo) ? -spl->bits : spl->bits, f);
 16   pack_mputw(spl->freq, f);
 17   pack_mputl(spl->len, f);
 18
 19   if (spl->bits == 8) {
 20      pack_fwrite(spl->data, spl->len * ((spl->stereo) ? 2 : 1), f);
 21   }
 22   else {
 23      int i;
 24
 25      for (i=0; i < (int)spl->len * ((spl->stereo) ? 2 : 1); i++) {
 26   pack_iputw(((int16_t *)spl->data)<i>, f);
 27      }
 28   }
 29
 30   if (*allegro_errno)
 31      return FALSE;
 32   else
 33      return TRUE;
 34   } /* save_sample_in_datafile( ... ) */
 35
 36
 37
 38/*!\fn save_libsound( LIBSOUND *lib , PACKFILE *file )
 39 *
 40 *\brief Save a sound library in an opened packfile
 41 *
 42 *\param lib The library to save
 43 *\param file The opened packfile where to save the library
 44 *
 45 *\return TRUE of FALSE
 46 */
 47
 48int save_libsound( LIBSOUND *lib , PACKFILE *file ) {
 49
 50    int it = 0, len = 0;
 51
 52    if( !lib || !file)
 53        return FALSE;
 54
 55    if( !lib -> name ){
 56        Malloc( lib -> name , char , uconvert_size( "defaut" , U_CURRENT, U_UNICODE) );
 57        ustrncpy( lib -> name , "defaut" , ustrlen( "defaut" ) );
 58    }
 59
 60    pack_iputl( lib -> current , file );
 61    pack_iputl( lib -> max , file );
 62    len = uconvert_size( lib -> name , U_CURRENT, U_UNICODE)
 63    pack_iputl( len , file );
 64    pack_fwrite( lib -> name , len , file );
 65
 66    for( it = 0 ; it < lib -> current ; it ++ ) {
 67        if( lib -> data[ it ] . state != NONE ){
 68            pack_iputl( 1 , file );
 69            pack_iputl( uconvert_size( lib -> data[ it ] . name , U_CURRENT, U_UNICODE)  , file );
 70            pack_fwrite( lib -> data[ it ] . name , uconvert_size( lib -> data[ it ] . name , U_CURRENT, U_UNICODE) , file );
 71            save_sample_in_datafile(  lib -> data[ it ] . sample , file);
 72    } else
 73            pack_iputl( 0 , file );
 74
 75    }
 76
 77    return TRUE;
 78
 79
 80} /* save_libsound( ... ) */

With this I got 129 Meg of wav files compressed into ... 145 Megs compressed datafile !

I must be wrong somewhere ...

If anyone have any idea of WTF

EDIT: Humpf I'm dumb. I'm just reding somethinh as create_lzss_pack_data in the manual... Am I on the right way ? ...

Fladimir da Gorf

Of course you could always use .zip files, for example. Allegro uses some crappy compression routines because they don't require additional libraries (like gz).

GullRaDriel

I am taking a look at zlib. My problem will be to manage the dynamic loading-freeing of songs into memory.

Maybe I will add a special flag into 'SOUND' and some test to see if i should load it, for how long, or if it is a 'permanent' sample.

Making ressources handler is a pain in the ass, but it's a necessary war !

dudaskank

I don't have that much experience in allegro packfiles, but if you try to compress some data that is already compressed, like mp3, xvid frames, jpeg, you get a bigger file. But I don't think that is the problem in this situation...

And, for a better compression ratio (better than zip) you can try the LZMA SDK. But the encoder is in C++...

GullRaDriel

I'll give it a try, even if I am better at C ;-p

umperio

Maybe it is not exactly what you want but you could look into physfs which is a nice resource handler.

http://icculus.org/physfs/

Regards

GullRaDriel

Umpf ! umperio...
That a great link !
It's free, downloadable, api, doc, binaries, src ! And even a tutorial.

Now the thread is OK :-p

Arthur Kalliokoski

Sound that's writtend directly to a file rarely compresses well, because the data isn't very redundant. If you encode the deltas between successive samples and compress those, it works much better. In other words, the sloping "side" of a sound wave is relatively constant. There's sound formats that already do this but I forget the name. PCM?

Kitty Cat

ADPCM, Adaptive Differential Pulse Code Modulation. PCM is the standard, uncompressed data.

Richard Phipps

If you have 129 meg of wav's than I'd look at using OGG files instead.

Evert

If you're interested in reading ZIP files, then there's an addon that allows you to do that. See http://www.allegro.cc/forums/thread/467091

Tobias Dammers

For large amounts of sound data, I'd strongly suggest you use some sort of lossy compression (ogg is good, mp3 has legal issues, wma even more so). Unless you do heavy processing on the audio, or use insane compression settings, you should be fine.
For music, you might also want to try tracker music instead of actual audio files; because of the typical repetitiveness of computer game music, tracker files are usually much smaller than audio files.
Finally, consider lowering audio quality. You don't need 96kHz, 24bit 5.1 samples. For a game, most sound effects can be 22.5 kHz, 16 bit, mono (some profit from being stereo though), without making much of an audible difference.

If all else fails, distribute on CD/DVD only

GullRaDriel

Richard said:

I'd look at using OGG files instead

My bad, I never achieve to make any ogg playing with alogg or dumb+ogg

Evert said:

If you're interested in reading ZIP files

Not for the same thing as the topic, but I take the link ;-)

Tobias said:

If all else fails, distribute on CD/DVD only

Since all the game/source/stuff is spread around the world by the magic of internet, it's not possible. And I don't have a kopeck to buy and send some copy.

HoHo

You could always provide two versions:
One with small size and compressed audio
Other with big size and uncompressed audio.

Perhaps you can even charge a few $ for the uncompressed thingie

Ron Novy

WavePack is a good lossless audio compression format but its very slow. It might be good for distributing large wav audio files that you don't want to compress into a lossy format. Just a note: the PACKFILE compresion uses an LZW style compresion that is not audio friendly. Audio data is to complex for LZW style compresion. Maybe its time to create a media friendly compression format that selects the compression based on the contents. Maybe a plug-in style compressor.

?Light bulb

typedef struct C_VTABLE {
   char *name;     /* name of plugin */
   char *author;   /* Name of author + email address + websit.... */
   char *file_ext; /* input extensions that might be relavent to this plug */
   int  type;      /* Type of data this plug in understands: TEXT=0, BIN=1, 
                    * BITMAP=2, AUDIO=3, VIDEO=4... 
                    */
   int  init_compressor(DC, format_info, ...);
   void *compress(DC, void *src, int size_in, int *size_out);
   int  init_decompressor(DC, format_info, ...);
   void *decompress(DC, void *src, int size_in, int *size_out);
} C_VTABLE;

GullRaDriel

Ron Novy, that's a nice start. I'm still waiting for the plug

Thread #582094. Printed from Allegro.cc