Changeset 147

Timestamp:

06/07/10 14:21:35 (3 years ago)

Author:

faltet

Message:

Supported a new parameter maxbytes for blosc_compress().

It represents a maximum of bytes for output. Tests unit added too.

Location:

trunk

Files:

• RELEASE_NOTES.txt (modified) (1 diff)
• bench/Makefile (added)
• bench/bench.c (modified) (4 diffs)
• src/blosc.c (modified) (18 diffs)
• src/blosc.h (modified) (4 diffs)
• tests/Makefile (modified) (1 diff)
• tests/test_all.sh (added)
• tests/test_api.c (modified) (3 diffs)
• tests/test_basics.c (added)

trunk/RELEASE_NOTES.txt

@@ -10 +10 @@
 ===========================
 
+- Supported a new parameter `maxbytes` for ``blosc_compress()``.  It
+  represents a maximum of bytes for output.  Tests unit added too.
+
 - Added 3 new functions for querying different metadata on compressed
   buffers.  A test suite for testing the new API has been added too.
-
 
 Changes from 0.9.3 to 0.9.4

trunk/bench/bench.c

@@ -143 +143 @@
 
 
-do_bench(int nthreads, unsigned int size, int elsize, int rshift) {
+void do_bench(int nthreads, unsigned int size, int elsize, int rshift) {
   void *src, *srccpy;
   void **dest[NCHUNKS], *dest2;
-  int nbytes, cbytes;
+  int nbytes = 0, cbytes = 0;
   size_t i, j;
   struct timeval last, current;
@@ -164 +164 @@
   memcpy(srccpy, src, size);
   for (j = 0; j < nchunks; j++) {
-    dest[j] = malloc(size);
+    /* 16 additional bytes should be enough for encoding everything */
+    dest[j] = malloc(size+16);
   }
 
@@ -210 +211 @@
     for (i = 0; i < niter; i++) {
       for (j = 0; j < nchunks; j++) {
-        cbytes = blosc_compress(clevel, doshuffle, elsize, size, src, dest[j]);
+        cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
+                                dest[j], size);
       }
     }
@@ -307 +309 @@
   char *usage = "Usage: bench ['single' | 'suite' | 'hardsuite' | 'extremesuite' | 'debugsuite'] [nthreads [bufsize(bytes) [typesize [sbits ]]]]";
 
+
+  if (argc == 1) {
+    printf("%s\n", usage);
+    exit(1);
+  }
 
   if (strcmp(argv[1], "single") == 0) {

trunk/src/blosc.c

@@ -87 +87 @@
   int32_t ntbytes;
   uint32_t nbytes;
+  uint32_t maxbytes;
   uint32_t nblocks;
   uint32_t leftover;
@@ -133 +134 @@
 /* Shuffle & compress a single block */
 static int blosc_c(size_t blocksize, int32_t leftoverblock,
-                   uint32_t ntbytes, uint32_t nbytes,
+                   uint32_t ntbytes, uint32_t maxbytes,
                    uint8_t *src, uint8_t *dest, uint8_t *tmp)
 {
@@ -168 +169 @@
     ctbytes += sizeof(int32_t);
     maxout = neblock;
-    if (ntbytes+maxout > nbytes) {
-      maxout = nbytes - ntbytes;   /* avoid buffer overrun */
+    if (ntbytes+maxout > maxbytes) {
+      maxout = maxbytes - ntbytes;   /* avoid buffer overrun */
       if (maxout <= 0) {
         return 0;                  /* non-compressible block */
@@ -188 +189 @@
       /* Before doing the copy, check that we are not running into a
          buffer overflow. */
-      if ((ntbytes+neblock) > nbytes) {
+      if ((ntbytes+neblock) > maxbytes) {
         return 0;    /* Non-compressible data */
       }
@@ -279 +280 @@
   size_t blocksize = params.blocksize;
   int32_t ntbytes = params.ntbytes;
-  int32_t nbytes = params.nbytes;
+  uint32_t maxbytes = params.maxbytes;
   uint32_t nblocks = params.nblocks;
   int32_t leftover = params.nbytes % params.blocksize;
@@ -299 +300 @@
     }
     if (compress) {
-      cbytes = blosc_c(bsize, leftoverblock, ntbytes, nbytes,
+      cbytes = blosc_c(bsize, leftoverblock, ntbytes, maxbytes,
                        src+j*blocksize, dest+ntbytes, tmp);
       if (cbytes == 0) {
@@ -311 +312 @@
     }
     if (cbytes < 0) {
-      ntbytes = cbytes;         /* error in blosc_c / blosc_d */
+      ntbytes = cbytes;         /* error in blosc_c or blosc_d */
       break;
     }
@@ -319 +320 @@
   /* Final check for ntbytes (only in compression mode) */
   if (compress) {
-    if (ntbytes == nbytes) {
+    if (ntbytes == maxbytes) {
       ntbytes = 0;               /* non-compressible data */
     }
-    else if (ntbytes > nbytes) {
+    else if (ntbytes > maxbytes) {
       fprintf(stderr, "The impossible happened: buffer overflow!\n");
       ntbytes = -5;               /* too large buffer */
@@ -520 +521 @@
 
 
+/* The public routine for compression.  See blosc.h for docstrings. */
 unsigned int blosc_compress(int clevel, int shuffle, size_t typesize,
-                            size_t nbytes, const void *src, void *dest)
+                            size_t nbytes, const void *src, void *dest,
+                            size_t maxbytes)
 {
   uint8_t *_dest=NULL;         /* current pos for destination buffer */
-  uint8_t *flags;              /* flags for header */
+  uint8_t *flags;              /* flags for header.  Currently booked:
+                                  - 0: shuffled?
+                                  - 1: memcpy'ed? */
   uint32_t nblocks;            /* number of total blocks in buffer */
   uint32_t leftover;           /* extra bytes at end of buffer */
@@ -543 +548 @@
   }
   else if (nbytes < MIN_BUFFERSIZE) {
-    /* Too little buffer.  Just return without doing anything else. */
+    /* Buffer too small.  Just return without doing anything else. */
     return 0;
   }
@@ -596 +601 @@
   params.ntbytes = ntbytes;
   params.nbytes = nbytes;
+  params.maxbytes = maxbytes;
   params.nblocks = nblocks;
   params.leftover = leftover;
@@ -604 +610 @@
   /* Do the actual compression */
   ntbytes = do_job();
+
+  /* Last chance for fitting `src` buffer in `dest` */
+  if ((ntbytes == 0) && (nbytes+16 <= maxbytes)) {
+    /* Specify that this buffer is memcpy'ed (bit 2 set to 1 in flags) */
+    _dest = (uint8_t *)(dest);
+    _dest[2] = 0x2;
+    memcpy(dest+16, src, nbytes);
+    ntbytes = nbytes+16;
+  }
+
   /* Set the number of compressed bytes in header */
   *ntbytes_ = sw32(ntbytes);
 
-  assert((int32_t)ntbytes < (int32_t)nbytes);
+  assert((int32_t)ntbytes <= (int32_t)maxbytes);
   return ntbytes;
 }
 
 
-unsigned int blosc_decompress(const void *src, void *dest, size_t dest_size)
+/* The public routine for decompression.  See blosc.h for docstrings. */
+unsigned int blosc_decompress(const void *src, void *dest, size_t destsize)
 {
   uint8_t *_src=NULL;            /* current pos for source buffer */
@@ -637 +654 @@
   blocksize = sw32(((uint32_t *)_src)[1]);   /* block size */
   ctbytes = sw32(((uint32_t *)_src)[2]);     /* compressed buffer size */
+
+  /* Check whether this buffer is memcpy'ed */
+  if (flags == 0x2) {
+    memcpy(dest, src+16, nbytes);
+    return nbytes;
+  }
+
   _src += sizeof(int32_t)*3;
   bstarts = (uint32_t *)_src;
@@ -647 +671 @@
 
   /* Check zero typesizes.  From Blosc version format 2 on, this value
-   has been reserved for future use (most probably to indicate
-   uncompressible buffers). */
+   has been reserved for future use. */
   if ((version == 1) && (typesize == 0)) {
     typesize = 256;             /* 0 means 256 in format version 1 */
   }
 
-  if (nbytes > dest_size) {
+  if (nbytes > destsize) {
     /* This should never happen but just in case */
     return -1;
@@ -680 +703 @@
   ntbytes = do_job();
 
-  assert(ntbytes <= (int32_t)dest_size);
+  assert(ntbytes <= (int32_t)destsize);
   return ntbytes;
 }
@@ -764 +787 @@
   size_t ebsize;
   int32_t compress;
-  uint32_t nbytes;
+  uint32_t maxbytes;
   uint32_t ntbytes;
   uint32_t nblocks;
@@ -815 +838 @@
     ebsize = blocksize + params.typesize*sizeof(int32_t);
     compress = params.compress;
-    nbytes = params.nbytes;
+    maxbytes = params.maxbytes;
     nblocks = params.nblocks;
     leftover = params.leftover;
@@ -887 +910 @@
         ntdest = params.ntbytes;
         bstarts[nblock_] = sw32(ntdest);    /* update block start counter */
-        if ( (cbytes == 0) || (ntdest+cbytes > (int32_t)nbytes) ) {
+        if ( (cbytes == 0) || (ntdest+cbytes > (int32_t)maxbytes) ) {
           giveup_code = 0;                  /* uncompressible buffer */
           pthread_mutex_unlock(&count_mutex);

trunk/src/blosc.h

@@ -57 +57 @@
   Only a typesize > 1 will allow the shuffle to work.
 
-  The `dest` buffer must have at least the size of the `src` buffer.
-  The `src` buffer and the `dest` buffer can not overlap.
+  The `dest` buffer must have at least the size of `maxbytes`.  Blosc
+  guarantees that if you set `maxbytes` at least to `nbytes` + 16, the
+  compression will always succeed.  The `src` buffer and the `dest`
+  buffer can not overlap.
 
-  If `src` buffer is not compressible (len(`dest`) >= len(`src`)), the
-  return value is zero and you should discard the contents of the
-  `dest` buffer.
+  If `src` buffer cannot be compressed into `maxbytes`, the return
+  value is zero and you should discard the contents of the `dest`
+  buffer.
 
   A negative return value means that an internal error happened.  This
@@ -69 +71 @@
 
   Compression is memory safe and guaranteed not to write the `dest`
-  buffer more than what is specified in `nbytes`.  However, it is not
-  re-entrant and not thread-safe (despite the fact that it uses
+  buffer more than what is specified in `maxbytes`.  However, it is
+  not re-entrant and not thread-safe (despite the fact that it uses
   threads internally ;-)
 
@@ -76 +78 @@
 
 unsigned int blosc_compress(int clevel, int doshuffle, size_t typesize,
-                            size_t nbytes, const void *src, void *dest);
+                            size_t nbytes, const void *src, void *dest,
+                            size_t maxbytes);
 
 
@@ -96 +99 @@
 */
 
-unsigned int blosc_decompress(const void *src, void *dest, size_t dest_size);
+unsigned int blosc_decompress(const void *src, void *dest, size_t destsize);
 
 

trunk/tests/Makefile

@@ -2 +2 @@
 CFLAGS=-O3 -msse2 -Wall
 LDFLAGS=-lpthread
-SOURCES=test_api.c ../src/blosc.c ../src/blosclz.c ../src/shuffle.c
-EXECUTABLE=test_api
+BLOSC_LIB= ../src/blosc.c ../src/blosclz.c ../src/shuffle.c
 
-all: $(SOURCES) $(EXECUTABLE)
+# The list of executables
+# Generated PNG (intermediate) files
+SOURCES := $(wildcard *.c)
+EXECUTABLES := $(patsubst %.c, %.exe, $(SOURCES))
 
-$(EXECUTABLE): $(SOURCES)
-        $(CC) $(CFLAGS) $(LDFLAGS) $(SOURCES) -o $@
+.PHONY: all
+all: test
+
+test: $(EXECUTABLES)
+
+%.exe: %.c $(BLOSC_LIB)
+        $(CC) $(CFLAGS) $(LDFLAGS) "$<" $(BLOSC_LIB) -o "$@"
 
 clean:
-        rm -rf test_api
+        rm -rf $(EXECUTABLES)

trunk/tests/test_api.c

@@ -10 +10 @@
 **********************************************************************/
 
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#if defined(_WIN32) && !defined(__MINGW32__)
-  #include <time.h>
-#else
-  #include <unistd.h>
-  #include <sys/time.h>
-#endif
-#include <math.h>
-#include "../src/blosc.h"
-
-
-/* This is MinUnit in action (http://www.jera.com/techinfo/jtns/jtn002.html) */
-#define mu_assert(message, test) do { if (!(test)) return message; } while (0)
-#define mu_run_test(test) do \
-    { char *message = test(); tests_run++;                          \
-      if (message) { printf("%c", 'F'); return message;}            \
-      else printf("%c", '.'); } while (0)
+#include "tests_blosc.h"
 
 int tests_run = 0;
-
-#define KB  1024
-#define MB  (1024*KB)
-#define GB  (1024*MB)
 
 /* Global vars */
@@ -101 +76 @@
 
   /* Get a compressed buffer */
-  cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest);
+  cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size);
 
   /* Get a decompressed buffer */
@@ -112 +87 @@
   }
   else {
-    printf("\nALL TESTS PASSED\n");
+    printf("\nALL TESTS PASSED for %s", argv[0]);
   }
-  printf("\nTests run: %d\n", tests_run);
+  printf("\tTests run: %d\n", tests_run);
 
+  free(src); free(srccpy); free(dest); free(dest2);
   return result != 0;
 }