Changeset 147


Ignore:
Timestamp:
06/07/10 14:21:35 (4 years ago)
Author:
faltet
Message:

Supported a new parameter maxbytes for blosc_compress().

It represents a maximum of bytes for output. Tests unit added too.

Location:
trunk
Files:
3 added
6 edited

Legend:

Unmodified
Added
Removed
  • trunk/RELEASE_NOTES.txt

    r146 r147  
    1010=========================== 
    1111 
     12- Supported a new parameter `maxbytes` for ``blosc_compress()``.  It 
     13  represents a maximum of bytes for output.  Tests unit added too. 
     14 
    1215- Added 3 new functions for querying different metadata on compressed 
    1316  buffers.  A test suite for testing the new API has been added too. 
    14  
    1517 
    1618Changes from 0.9.3 to 0.9.4 
  • trunk/bench/bench.c

    r142 r147  
    143143 
    144144 
    145 do_bench(int nthreads, unsigned int size, int elsize, int rshift) { 
     145void do_bench(int nthreads, unsigned int size, int elsize, int rshift) { 
    146146  void *src, *srccpy; 
    147147  void **dest[NCHUNKS], *dest2; 
    148   int nbytes, cbytes; 
     148  int nbytes = 0, cbytes = 0; 
    149149  size_t i, j; 
    150150  struct timeval last, current; 
     
    164164  memcpy(srccpy, src, size); 
    165165  for (j = 0; j < nchunks; j++) { 
    166     dest[j] = malloc(size); 
     166    /* 16 additional bytes should be enough for encoding everything */ 
     167    dest[j] = malloc(size+16); 
    167168  } 
    168169 
     
    210211    for (i = 0; i < niter; i++) { 
    211212      for (j = 0; j < nchunks; j++) { 
    212         cbytes = blosc_compress(clevel, doshuffle, elsize, size, src, dest[j]); 
     213        cbytes = blosc_compress(clevel, doshuffle, elsize, size, src, 
     214                                dest[j], size); 
    213215      } 
    214216    } 
     
    307309  char *usage = "Usage: bench ['single' | 'suite' | 'hardsuite' | 'extremesuite' | 'debugsuite'] [nthreads [bufsize(bytes) [typesize [sbits ]]]]"; 
    308310 
     311 
     312  if (argc == 1) { 
     313    printf("%s\n", usage); 
     314    exit(1); 
     315  } 
    309316 
    310317  if (strcmp(argv[1], "single") == 0) { 
  • trunk/src/blosc.c

    r146 r147  
    8787  int32_t ntbytes; 
    8888  uint32_t nbytes; 
     89  uint32_t maxbytes; 
    8990  uint32_t nblocks; 
    9091  uint32_t leftover; 
     
    133134/* Shuffle & compress a single block */ 
    134135static int blosc_c(size_t blocksize, int32_t leftoverblock, 
    135                    uint32_t ntbytes, uint32_t nbytes, 
     136                   uint32_t ntbytes, uint32_t maxbytes, 
    136137                   uint8_t *src, uint8_t *dest, uint8_t *tmp) 
    137138{ 
     
    168169    ctbytes += sizeof(int32_t); 
    169170    maxout = neblock; 
    170     if (ntbytes+maxout > nbytes) { 
    171       maxout = nbytes - ntbytes;   /* avoid buffer overrun */ 
     171    if (ntbytes+maxout > maxbytes) { 
     172      maxout = maxbytes - ntbytes;   /* avoid buffer overrun */ 
    172173      if (maxout <= 0) { 
    173174        return 0;                  /* non-compressible block */ 
     
    188189      /* Before doing the copy, check that we are not running into a 
    189190         buffer overflow. */ 
    190       if ((ntbytes+neblock) > nbytes) { 
     191      if ((ntbytes+neblock) > maxbytes) { 
    191192        return 0;    /* Non-compressible data */ 
    192193      } 
     
    279280  size_t blocksize = params.blocksize; 
    280281  int32_t ntbytes = params.ntbytes; 
    281   int32_t nbytes = params.nbytes; 
     282  uint32_t maxbytes = params.maxbytes; 
    282283  uint32_t nblocks = params.nblocks; 
    283284  int32_t leftover = params.nbytes % params.blocksize; 
     
    299300    } 
    300301    if (compress) { 
    301       cbytes = blosc_c(bsize, leftoverblock, ntbytes, nbytes, 
     302      cbytes = blosc_c(bsize, leftoverblock, ntbytes, maxbytes, 
    302303                       src+j*blocksize, dest+ntbytes, tmp); 
    303304      if (cbytes == 0) { 
     
    311312    } 
    312313    if (cbytes < 0) { 
    313       ntbytes = cbytes;         /* error in blosc_c / blosc_d */ 
     314      ntbytes = cbytes;         /* error in blosc_c or blosc_d */ 
    314315      break; 
    315316    } 
     
    319320  /* Final check for ntbytes (only in compression mode) */ 
    320321  if (compress) { 
    321     if (ntbytes == nbytes) { 
     322    if (ntbytes == maxbytes) { 
    322323      ntbytes = 0;               /* non-compressible data */ 
    323324    } 
    324     else if (ntbytes > nbytes) { 
     325    else if (ntbytes > maxbytes) { 
    325326      fprintf(stderr, "The impossible happened: buffer overflow!\n"); 
    326327      ntbytes = -5;               /* too large buffer */ 
     
    520521 
    521522 
     523/* The public routine for compression.  See blosc.h for docstrings. */ 
    522524unsigned int blosc_compress(int clevel, int shuffle, size_t typesize, 
    523                             size_t nbytes, const void *src, void *dest) 
     525                            size_t nbytes, const void *src, void *dest, 
     526                            size_t maxbytes) 
    524527{ 
    525528  uint8_t *_dest=NULL;         /* current pos for destination buffer */ 
    526   uint8_t *flags;              /* flags for header */ 
     529  uint8_t *flags;              /* flags for header.  Currently booked: 
     530                                  - 0: shuffled? 
     531                                  - 1: memcpy'ed? */ 
    527532  uint32_t nblocks;            /* number of total blocks in buffer */ 
    528533  uint32_t leftover;           /* extra bytes at end of buffer */ 
     
    543548  } 
    544549  else if (nbytes < MIN_BUFFERSIZE) { 
    545     /* Too little buffer.  Just return without doing anything else. */ 
     550    /* Buffer too small.  Just return without doing anything else. */ 
    546551    return 0; 
    547552  } 
     
    596601  params.ntbytes = ntbytes; 
    597602  params.nbytes = nbytes; 
     603  params.maxbytes = maxbytes; 
    598604  params.nblocks = nblocks; 
    599605  params.leftover = leftover; 
     
    604610  /* Do the actual compression */ 
    605611  ntbytes = do_job(); 
     612 
     613  /* Last chance for fitting `src` buffer in `dest` */ 
     614  if ((ntbytes == 0) && (nbytes+16 <= maxbytes)) { 
     615    /* Specify that this buffer is memcpy'ed (bit 2 set to 1 in flags) */ 
     616    _dest = (uint8_t *)(dest); 
     617    _dest[2] = 0x2; 
     618    memcpy(dest+16, src, nbytes); 
     619    ntbytes = nbytes+16; 
     620  } 
     621 
    606622  /* Set the number of compressed bytes in header */ 
    607623  *ntbytes_ = sw32(ntbytes); 
    608624 
    609   assert((int32_t)ntbytes < (int32_t)nbytes); 
     625  assert((int32_t)ntbytes <= (int32_t)maxbytes); 
    610626  return ntbytes; 
    611627} 
    612628 
    613629 
    614 unsigned int blosc_decompress(const void *src, void *dest, size_t dest_size) 
     630/* The public routine for decompression.  See blosc.h for docstrings. */ 
     631unsigned int blosc_decompress(const void *src, void *dest, size_t destsize) 
    615632{ 
    616633  uint8_t *_src=NULL;            /* current pos for source buffer */ 
     
    637654  blocksize = sw32(((uint32_t *)_src)[1]);   /* block size */ 
    638655  ctbytes = sw32(((uint32_t *)_src)[2]);     /* compressed buffer size */ 
     656 
     657  /* Check whether this buffer is memcpy'ed */ 
     658  if (flags == 0x2) { 
     659    memcpy(dest, src+16, nbytes); 
     660    return nbytes; 
     661  } 
     662 
    639663  _src += sizeof(int32_t)*3; 
    640664  bstarts = (uint32_t *)_src; 
     
    647671 
    648672  /* Check zero typesizes.  From Blosc version format 2 on, this value 
    649    has been reserved for future use (most probably to indicate 
    650    uncompressible buffers). */ 
     673   has been reserved for future use. */ 
    651674  if ((version == 1) && (typesize == 0)) { 
    652675    typesize = 256;             /* 0 means 256 in format version 1 */ 
    653676  } 
    654677 
    655   if (nbytes > dest_size) { 
     678  if (nbytes > destsize) { 
    656679    /* This should never happen but just in case */ 
    657680    return -1; 
     
    680703  ntbytes = do_job(); 
    681704 
    682   assert(ntbytes <= (int32_t)dest_size); 
     705  assert(ntbytes <= (int32_t)destsize); 
    683706  return ntbytes; 
    684707} 
     
    764787  size_t ebsize; 
    765788  int32_t compress; 
    766   uint32_t nbytes; 
     789  uint32_t maxbytes; 
    767790  uint32_t ntbytes; 
    768791  uint32_t nblocks; 
     
    815838    ebsize = blocksize + params.typesize*sizeof(int32_t); 
    816839    compress = params.compress; 
    817     nbytes = params.nbytes; 
     840    maxbytes = params.maxbytes; 
    818841    nblocks = params.nblocks; 
    819842    leftover = params.leftover; 
     
    887910        ntdest = params.ntbytes; 
    888911        bstarts[nblock_] = sw32(ntdest);    /* update block start counter */ 
    889         if ( (cbytes == 0) || (ntdest+cbytes > (int32_t)nbytes) ) { 
     912        if ( (cbytes == 0) || (ntdest+cbytes > (int32_t)maxbytes) ) { 
    890913          giveup_code = 0;                  /* uncompressible buffer */ 
    891914          pthread_mutex_unlock(&count_mutex); 
  • trunk/src/blosc.h

    r146 r147  
    5757  Only a typesize > 1 will allow the shuffle to work. 
    5858 
    59   The `dest` buffer must have at least the size of the `src` buffer. 
    60   The `src` buffer and the `dest` buffer can not overlap. 
     59  The `dest` buffer must have at least the size of `maxbytes`.  Blosc 
     60  guarantees that if you set `maxbytes` at least to `nbytes` + 16, the 
     61  compression will always succeed.  The `src` buffer and the `dest` 
     62  buffer can not overlap. 
    6163 
    62   If `src` buffer is not compressible (len(`dest`) >= len(`src`)), the 
    63   return value is zero and you should discard the contents of the 
    64   `dest` buffer. 
     64  If `src` buffer cannot be compressed into `maxbytes`, the return 
     65  value is zero and you should discard the contents of the `dest` 
     66  buffer. 
    6567 
    6668  A negative return value means that an internal error happened.  This 
     
    6971 
    7072  Compression is memory safe and guaranteed not to write the `dest` 
    71   buffer more than what is specified in `nbytes`.  However, it is not 
    72   re-entrant and not thread-safe (despite the fact that it uses 
     73  buffer more than what is specified in `maxbytes`.  However, it is 
     74  not re-entrant and not thread-safe (despite the fact that it uses 
    7375  threads internally ;-) 
    7476 
     
    7678 
    7779unsigned int blosc_compress(int clevel, int doshuffle, size_t typesize, 
    78                             size_t nbytes, const void *src, void *dest); 
     80                            size_t nbytes, const void *src, void *dest, 
     81                            size_t maxbytes); 
    7982 
    8083 
     
    9699*/ 
    97100 
    98 unsigned int blosc_decompress(const void *src, void *dest, size_t dest_size); 
     101unsigned int blosc_decompress(const void *src, void *dest, size_t destsize); 
    99102 
    100103 
  • trunk/tests/Makefile

    r146 r147  
    22CFLAGS=-O3 -msse2 -Wall 
    33LDFLAGS=-lpthread 
    4 SOURCES=test_api.c ../src/blosc.c ../src/blosclz.c ../src/shuffle.c 
    5 EXECUTABLE=test_api 
     4BLOSC_LIB= ../src/blosc.c ../src/blosclz.c ../src/shuffle.c 
    65 
    7 all: $(SOURCES) $(EXECUTABLE) 
     6# The list of executables 
     7# Generated PNG (intermediate) files 
     8SOURCES := $(wildcard *.c) 
     9EXECUTABLES := $(patsubst %.c, %.exe, $(SOURCES)) 
    810 
    9 $(EXECUTABLE): $(SOURCES) 
    10         $(CC) $(CFLAGS) $(LDFLAGS) $(SOURCES) -o $@ 
     11.PHONY: all 
     12all: test 
     13 
     14test: $(EXECUTABLES) 
     15 
     16%.exe: %.c $(BLOSC_LIB) 
     17        $(CC) $(CFLAGS) $(LDFLAGS) "$<" $(BLOSC_LIB) -o "$@" 
    1118 
    1219clean: 
    13         rm -rf test_api 
     20        rm -rf $(EXECUTABLES) 
  • trunk/tests/test_api.c

    r146 r147  
    1010**********************************************************************/ 
    1111 
    12 #include <stdlib.h> 
    13 #include <stdio.h> 
    14 #include <string.h> 
    15 #include <sys/types.h> 
    16 #include <sys/stat.h> 
    17 #include <fcntl.h> 
    18 #if defined(_WIN32) && !defined(__MINGW32__) 
    19   #include <time.h> 
    20 #else 
    21   #include <unistd.h> 
    22   #include <sys/time.h> 
    23 #endif 
    24 #include <math.h> 
    25 #include "../src/blosc.h" 
    26  
    27  
    28 /* This is MinUnit in action (http://www.jera.com/techinfo/jtns/jtn002.html) */ 
    29 #define mu_assert(message, test) do { if (!(test)) return message; } while (0) 
    30 #define mu_run_test(test) do \ 
    31     { char *message = test(); tests_run++;                          \ 
    32       if (message) { printf("%c", 'F'); return message;}            \ 
    33       else printf("%c", '.'); } while (0) 
     12#include "tests_blosc.h" 
    3413 
    3514int tests_run = 0; 
    36  
    37 #define KB  1024 
    38 #define MB  (1024*KB) 
    39 #define GB  (1024*MB) 
    4015 
    4116/* Global vars */ 
     
    10176 
    10277  /* Get a compressed buffer */ 
    103   cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest); 
     78  cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size); 
    10479 
    10580  /* Get a decompressed buffer */ 
     
    11287  } 
    11388  else { 
    114     printf("\nALL TESTS PASSED\n"); 
     89    printf("\nALL TESTS PASSED for %s", argv[0]); 
    11590  } 
    116   printf("\nTests run: %d\n", tests_run); 
     91  printf("\tTests run: %d\n", tests_run); 
    11792 
     93  free(src); free(srccpy); free(dest); free(dest2); 
    11894  return result != 0; 
    11995} 
Note: See TracChangeset for help on using the changeset viewer.