source: trunk/bench/bench.c @ 148

Revision 148, 12.7 KB checked in by faltet, 4 years ago (diff)

Introduced BLOSC_MAX_OVERHEAD symbol that is the maximum overhead introduced by Blosc during compression.

Line 
1/*********************************************************************
2  Small benchmark for testing basic capabilities of Blosc.
3
4  You can select different degrees of 'randomness' in input buffer, as
5  well as external datafiles (uncomment the lines after "For data
6  coming from a file" comment).
7
8  To compile using GCC, stay in this directory and do:
9
10    gcc -O3 -msse2 -o bench bench.c \
11        ../src/blosc.c ../src/blosclz.c ../src/shuffle.c -lpthread
12
13  I'm collecting speeds for different machines, so the output of your
14  benchmarks and your processor specifications are welcome!
15
16  Author: Francesc Alted (faltet@pytables.org)
17
18  See LICENSES/BLOSC.txt for details about copyright and rights to use.
19**********************************************************************/
20
21#include <stdlib.h>
22#include <stdio.h>
23#include <string.h>
24#include <sys/types.h>
25#include <sys/stat.h>
26#include <fcntl.h>
27#if defined(_WIN32) && !defined(__MINGW32__)
28  #include <time.h>
29#else
30  #include <unistd.h>
31  #include <sys/time.h>
32#endif
33#include <math.h>
34#include "../src/blosc.h"
35
36#define KB  1024
37#define MB  (1024*KB)
38#define GB  (1024*MB)
39
40#define NCHUNKS (32*1024)       /* maximum number of chunks */
41
42
43int nchunks = NCHUNKS;
44int niter = 3;                  /* default number of iterations */
45float totalsize = 0.;           /* total compressed/decompressed size */
46
47#if defined(_WIN32) && !defined(__MINGW32__)
48#include <windows.h>
49#if defined(_MSC_VER) || defined(_MSC_EXTENSIONS)
50  #define DELTA_EPOCH_IN_MICROSECS  11644473600000000Ui64
51#else
52  #define DELTA_EPOCH_IN_MICROSECS  11644473600000000ULL
53#endif
54
55struct timezone
56{
57  int  tz_minuteswest; /* minutes W of Greenwich */
58  int  tz_dsttime;     /* type of dst correction */
59};
60
61int gettimeofday(struct timeval *tv, struct timezone *tz)
62{
63  FILETIME ft;
64  unsigned __int64 tmpres = 0;
65  static int tzflag;
66
67  if (NULL != tv)
68  {
69    GetSystemTimeAsFileTime(&ft);
70
71    tmpres |= ft.dwHighDateTime;
72    tmpres <<= 32;
73    tmpres |= ft.dwLowDateTime;
74
75    /*converting file time to unix epoch*/
76    tmpres -= DELTA_EPOCH_IN_MICROSECS;
77    tmpres /= 10;  /*convert into microseconds*/
78    tv->tv_sec = (long)(tmpres / 1000000UL);
79    tv->tv_usec = (long)(tmpres % 1000000UL);
80  }
81
82  if (NULL != tz)
83  {
84    if (!tzflag)
85    {
86      _tzset();
87      tzflag++;
88    }
89    tz->tz_minuteswest = _timezone / 60;
90    tz->tz_dsttime = _daylight;
91  }
92
93  return 0;
94}
95#endif   /* _WIN32 */
96
97
98/* Given two timeval stamps, return the difference in seconds */
99float getseconds(struct timeval last, struct timeval current) {
100  int sec, usec;
101
102  sec = current.tv_sec - last.tv_sec;
103  usec = current.tv_usec - last.tv_usec;
104  return (float)(((double)sec + usec*1e-6));
105}
106
107/* Given two timeval stamps, return the time per chunk in usec */
108float get_usec_chunk(struct timeval last, struct timeval current) {
109  return (float)(getseconds(last, current)/(niter*nchunks)*1e6);
110}
111
112
113int get_value(int i, int rshift) {
114  int v;
115
116  v = (i<<26)^(i<<18)^(i<<11)^(i<<3)^i;
117  if (rshift < 32) {
118    v &= (1 << rshift) - 1;
119  }
120  return v;
121}
122
123
124void init_buffer(void *src, int size, int rshift) {
125  unsigned int i;
126  int *_src = (int *)src;
127
128  /* To have reproducible results */
129  srand(1);
130
131  /* Initialize the original buffer */
132  for (i = 0; i < size/sizeof(int); ++i) {
133    /* Choose one below */
134    //_src[i] = 0;
135    //_src[i] = 0x01010101;
136    //_src[i] = 0x01020304;
137    //_src[i] = i * 1/.3;
138    //_src[i] = i;
139    //_src[i] = rand() >> (32-rshift);
140    _src[i] = get_value(i, rshift);
141  }
142}
143
144
145void do_bench(int nthreads, unsigned int size, int elsize, int rshift) {
146  void *src, *srccpy;
147  void **dest[NCHUNKS], *dest2;
148  int nbytes = 0, cbytes = 0;
149  size_t i, j;
150  struct timeval last, current;
151  float tmemcpy, tshuf, tunshuf;
152  int clevel, doshuffle=1;
153  unsigned char *orig, *round;
154
155  blosc_set_nthreads(nthreads);
156
157  /* Initialize buffers */
158  src = malloc(size);
159  srccpy = malloc(size);
160  dest2 = malloc(size);
161  /* zero src to initialize byte on it, and not only multiples of 4 */
162  memset(src, 0, size);
163  init_buffer(src, size, rshift);
164  memcpy(srccpy, src, size);
165  for (j = 0; j < nchunks; j++) {
166    dest[j] = malloc(size+BLOSC_MAX_OVERHEAD);
167  }
168
169  /* Warm destination memory (memcpy() will go a bit faster later on) */
170  for (j = 0; j < nchunks; j++) {
171    memcpy(dest[j], src, size);
172  }
173
174  printf("--> %d, %d, %d, %d\n", nthreads, size, elsize, rshift);
175  printf("********************** Run info ******************************\n");
176  printf("Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
177  printf("Using synthetic data with %d significant bits (out of 32)\n", rshift);
178  printf("Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
179  printf("Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
180  printf("Number of threads: %d\n", nthreads);
181  printf("********************** Running benchmarks *********************\n");
182
183  gettimeofday(&last, NULL);
184  for (i = 0; i < niter; i++) {
185    for (j = 0; j < nchunks; j++) {
186      memcpy(dest[j], src, size);
187    }
188  }
189  gettimeofday(&current, NULL);
190  tmemcpy = get_usec_chunk(last, current);
191  printf("memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
192         tmemcpy, size/(tmemcpy*MB/1e6));
193
194  gettimeofday(&last, NULL);
195  for (i = 0; i < niter; i++) {
196    for (j = 0; j < nchunks; j++) {
197      memcpy(dest2, dest[j], size);
198    }
199  }
200  gettimeofday(&current, NULL);
201  tmemcpy = get_usec_chunk(last, current);
202  printf("memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
203         tmemcpy, size/(tmemcpy*MB/1e6));
204
205  for (clevel=1; clevel<10; clevel++) {
206
207    printf("Compression level: %d\n", clevel);
208
209    gettimeofday(&last, NULL);
210    for (i = 0; i < niter; i++) {
211      for (j = 0; j < nchunks; j++) {
212        cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
213                                dest[j], size+BLOSC_MAX_OVERHEAD);
214      }
215    }
216    gettimeofday(&current, NULL);
217    tshuf = get_usec_chunk(last, current);
218    printf("comp(write):\t %6.1f us, %.1f MB/s\t  ",
219           tshuf, size/(tshuf*MB/1e6));
220    printf("Final bytes: %d  ", cbytes);
221    if (cbytes > 0) {
222      printf("Ratio: %3.2f", size/(float)cbytes);
223    }
224    printf("\n");
225
226    /* Compressor was unable to compress.  Copy the buffer manually. */
227    if (cbytes == 0) {
228      for (j = 0; j < nchunks; j++) {
229        memcpy(dest[j], src, size);
230      }
231    }
232
233    gettimeofday(&last, NULL);
234    for (i = 0; i < niter; i++) {
235      for (j = 0; j < nchunks; j++) {
236        if (cbytes == 0) {
237          memcpy(dest2, dest[j], size);
238          nbytes = size;
239        }
240        else {
241          nbytes = blosc_decompress(dest[j], dest2, size);
242        }
243      }
244    }
245    gettimeofday(&current, NULL);
246    tunshuf = get_usec_chunk(last, current);
247    printf("decomp(read):\t %6.1f us, %.1f MB/s\t  ",
248           tunshuf, nbytes/(tunshuf*MB/1e6));
249    if (nbytes < 0) {
250      printf("FAILED.  Error code: %d\n", nbytes);
251    }
252    /* printf("Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
253
254    /* Check if data has had a good roundtrip */
255    orig = (unsigned char *)srccpy;
256    round = (unsigned char *)dest2;
257    for(i = 0; i<size; ++i){
258      if (orig[i] != round[i]) {
259        printf("\nError: Original data and round-trip do not match in pos %d\n",
260               (int)i);
261        printf("Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
262        break;
263      }
264    }
265
266    if (i == size) printf("OK\n");
267
268  } /* End clevel loop */
269
270
271  /* To compute the totalsize, we should take into account the 9
272     compression levels */
273  totalsize += (size * nchunks * niter * 9.);
274
275  free(src); free(srccpy); free(dest2);
276  for (i = 0; i < nchunks; i++) {
277    free(dest[i]);
278  }
279
280}
281
282
283/* Compute a sensible value for nchunks */
284int get_nchunks(int size_, int ws) {
285  int nchunks;
286
287  nchunks = ws / size_;
288  if (nchunks > NCHUNKS) nchunks = NCHUNKS;
289  if (nchunks < 1) nchunks = 1;
290  return nchunks;
291}
292
293
294int main(int argc, char *argv[]) {
295  int single = 1;
296  int suite = 0;
297  int hard_suite = 0;
298  int extreme_suite = 0;
299  int debug_suite = 0;
300  int nthreads = 1;                     /* The number of threads */
301  int size = 2*MB;                      /* Buffer size */
302  int elsize = 8;                       /* Datatype size */
303  int rshift = 19;                      /* Significant bits */
304  int workingset = 256*MB;              /* The maximum allocated memory */
305  int nthreads_, size_, elsize_, rshift_, i;
306  struct timeval last, current;
307  float totaltime;
308  char *usage = "Usage: bench ['single' | 'suite' | 'hardsuite' | 'extremesuite' | 'debugsuite'] [nthreads [bufsize(bytes) [typesize [sbits ]]]]";
309
310
311  if (argc == 1) {
312    printf("%s\n", usage);
313    exit(1);
314  }
315
316  if (strcmp(argv[1], "single") == 0) {
317    single = 1;
318  }
319  else if (strcmp(argv[1], "suite") == 0) {
320    suite = 1;
321  }
322  else if (strcmp(argv[1], "hardsuite") == 0) {
323    hard_suite = 1;
324    workingset = 64*MB;
325    /* Values here are ending points for loops */
326    nthreads = 2;
327    size = 8*MB;
328    elsize = 32;
329    rshift = 32;
330  }
331  else if (strcmp(argv[1], "extremesuite") == 0) {
332    extreme_suite = 1;
333    workingset = 32*MB;
334    niter = 1;
335    /* Values here are ending points for loops */
336    nthreads = 4;
337    size = 16*MB;
338    elsize = 32;
339    rshift = 32;
340  }
341  else if (strcmp(argv[1], "debugsuite") == 0) {
342    debug_suite = 1;
343    workingset = 32*MB;
344    niter = 1;
345    /* Warning: values here are starting points for loops.  This is
346       useful for debugging. */
347    nthreads = 1;
348    size = 16*KB;
349    elsize = 1;
350    rshift = 0;
351  }
352  else {
353    printf("%s\n", usage);
354    exit(1);
355  }
356
357  if (argc >= 3) {
358    nthreads = atoi(argv[2]);
359  }
360  if (argc >= 4) {
361    size = atoi(argv[3]);
362  }
363  if (argc >= 5) {
364    elsize = atoi(argv[4]);
365  }
366  if (argc >= 6) {
367    rshift = atoi(argv[5]);
368  }
369
370  if ((argc >= 7) || !(single || suite || hard_suite || extreme_suite)) {
371    printf("%s\n", usage);
372    exit(1);
373  }
374
375  nchunks = get_nchunks(size, workingset);
376  gettimeofday(&last, NULL);
377
378  if (suite) {
379    for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) {
380        do_bench(nthreads_, size, elsize, rshift);
381    }
382  }
383  else if (hard_suite) {
384    for (rshift_ = 0; rshift_ <= rshift; rshift_ += 5) {
385      for (elsize_ = 1; elsize_ <= elsize; elsize_ *= 2) {
386        /* The next loop is for getting sizes that are not power of 2 */
387        for (i = -elsize_; i <= elsize_; i += elsize_) {
388          for (size_ = 32*KB; size_ <= size; size_ *= 2) {
389            nchunks = get_nchunks(size_+i, workingset);
390            niter = 1;
391            for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
392              do_bench(nthreads_, size_+i, elsize_, rshift_);
393              gettimeofday(&current, NULL);
394              totaltime = getseconds(last, current);
395              printf("Elapsed time:\t %6.1f s.  Processed data: %.1f GB\n",
396                     totaltime, totalsize / GB);
397            }
398          }
399        }
400      }
401    }
402  }
403  else if (extreme_suite) {
404    for (rshift_ = 0; rshift_ <= rshift; rshift_++) {
405      for (elsize_ = 1; elsize_ <= elsize; elsize_++) {
406        /* The next loop is for getting sizes that are not power of 2 */
407        for (i = -elsize_*2; i <= elsize_*2; i += elsize_) {
408          for (size_ = 32*KB; size_ <= size; size_ *= 2) {
409            nchunks = get_nchunks(size_+i, workingset);
410            for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
411              do_bench(nthreads_, size_+i, elsize_, rshift_);
412              gettimeofday(&current, NULL);
413              totaltime = getseconds(last, current);
414              printf("Elapsed time:\t %6.1f s.  Processed data: %.1f GB\n",
415                     totaltime, totalsize / GB);
416            }
417          }
418        }
419      }
420    }
421  }
422  else if (debug_suite) {
423    for (rshift_ = rshift; rshift_ <= 32; rshift_++) {
424      for (elsize_ = elsize; elsize_ <= 32; elsize_++) {
425        /* The next loop is for getting sizes that are not power of 2 */
426        for (i = -elsize_*2; i <= elsize_*2; i += elsize_) {
427          for (size_ = size; size_ <= 16*MB; size_ *= 2) {
428            nchunks = get_nchunks(size_+i, workingset);
429            for (nthreads_ = nthreads; nthreads_ <= 4; nthreads_++) {
430              do_bench(nthreads_, size_+i, elsize_, rshift_);
431              gettimeofday(&current, NULL);
432              totaltime = getseconds(last, current);
433              printf("Elapsed time:\t %6.1f s.  Processed data: %.1f GB\n",
434                     totaltime, totalsize / GB);
435            }
436          }
437        }
438      }
439    }
440  }
441  /* Single mode */
442  else {
443    do_bench(nthreads, size, elsize, rshift);
444  }
445
446  /* Print out some statistics */
447  gettimeofday(&current, NULL);
448  totaltime = getseconds(last, current);
449  printf("\nRound-trip compr/decompr on %.1f GB\n", totalsize / GB);
450  printf("Elapsed time:\t %6.1f s, %.1f MB/s\n",
451         totaltime, totalsize*2*1.1/(MB*totaltime));
452
453  /* Free blosc resources */
454  blosc_free_resources();
455
456  return 0;
457}
Note: See TracBrowser for help on using the repository browser.