source: trunk/bench/bench.c @ 147

Revision 147, 12.7 KB checked in by faltet, 4 years ago (diff)

Supported a new parameter maxbytes for blosc_compress().

It represents a maximum of bytes for output. Tests unit added too.

Line 
1/*********************************************************************
2  Small benchmark for testing basic capabilities of Blosc.
3
4  You can select different degrees of 'randomness' in input buffer, as
5  well as external datafiles (uncomment the lines after "For data
6  coming from a file" comment).
7
8  To compile using GCC, stay in this directory and do:
9
10    gcc -O3 -msse2 -o bench bench.c \
11        ../src/blosc.c ../src/blosclz.c ../src/shuffle.c -lpthread
12
13  I'm collecting speeds for different machines, so the output of your
14  benchmarks and your processor specifications are welcome!
15
16  Author: Francesc Alted (faltet@pytables.org)
17
18  See LICENSES/BLOSC.txt for details about copyright and rights to use.
19**********************************************************************/
20
21#include <stdlib.h>
22#include <stdio.h>
23#include <string.h>
24#include <sys/types.h>
25#include <sys/stat.h>
26#include <fcntl.h>
27#if defined(_WIN32) && !defined(__MINGW32__)
28  #include <time.h>
29#else
30  #include <unistd.h>
31  #include <sys/time.h>
32#endif
33#include <math.h>
34#include "../src/blosc.h"
35
36#define KB  1024
37#define MB  (1024*KB)
38#define GB  (1024*MB)
39
40#define NCHUNKS (32*1024)       /* maximum number of chunks */
41
42
43int nchunks = NCHUNKS;
44int niter = 3;                  /* default number of iterations */
45float totalsize = 0.;           /* total compressed/decompressed size */
46
47#if defined(_WIN32) && !defined(__MINGW32__)
48#include <windows.h>
49#if defined(_MSC_VER) || defined(_MSC_EXTENSIONS)
50  #define DELTA_EPOCH_IN_MICROSECS  11644473600000000Ui64
51#else
52  #define DELTA_EPOCH_IN_MICROSECS  11644473600000000ULL
53#endif
54
55struct timezone
56{
57  int  tz_minuteswest; /* minutes W of Greenwich */
58  int  tz_dsttime;     /* type of dst correction */
59};
60
61int gettimeofday(struct timeval *tv, struct timezone *tz)
62{
63  FILETIME ft;
64  unsigned __int64 tmpres = 0;
65  static int tzflag;
66
67  if (NULL != tv)
68  {
69    GetSystemTimeAsFileTime(&ft);
70
71    tmpres |= ft.dwHighDateTime;
72    tmpres <<= 32;
73    tmpres |= ft.dwLowDateTime;
74
75    /*converting file time to unix epoch*/
76    tmpres -= DELTA_EPOCH_IN_MICROSECS;
77    tmpres /= 10;  /*convert into microseconds*/
78    tv->tv_sec = (long)(tmpres / 1000000UL);
79    tv->tv_usec = (long)(tmpres % 1000000UL);
80  }
81
82  if (NULL != tz)
83  {
84    if (!tzflag)
85    {
86      _tzset();
87      tzflag++;
88    }
89    tz->tz_minuteswest = _timezone / 60;
90    tz->tz_dsttime = _daylight;
91  }
92
93  return 0;
94}
95#endif   /* _WIN32 */
96
97
98/* Given two timeval stamps, return the difference in seconds */
99float getseconds(struct timeval last, struct timeval current) {
100  int sec, usec;
101
102  sec = current.tv_sec - last.tv_sec;
103  usec = current.tv_usec - last.tv_usec;
104  return (float)(((double)sec + usec*1e-6));
105}
106
107/* Given two timeval stamps, return the time per chunk in usec */
108float get_usec_chunk(struct timeval last, struct timeval current) {
109  return (float)(getseconds(last, current)/(niter*nchunks)*1e6);
110}
111
112
113int get_value(int i, int rshift) {
114  int v;
115
116  v = (i<<26)^(i<<18)^(i<<11)^(i<<3)^i;
117  if (rshift < 32) {
118    v &= (1 << rshift) - 1;
119  }
120  return v;
121}
122
123
124void init_buffer(void *src, int size, int rshift) {
125  unsigned int i;
126  int *_src = (int *)src;
127
128  /* To have reproducible results */
129  srand(1);
130
131  /* Initialize the original buffer */
132  for (i = 0; i < size/sizeof(int); ++i) {
133    /* Choose one below */
134    //_src[i] = 0;
135    //_src[i] = 0x01010101;
136    //_src[i] = 0x01020304;
137    //_src[i] = i * 1/.3;
138    //_src[i] = i;
139    //_src[i] = rand() >> (32-rshift);
140    _src[i] = get_value(i, rshift);
141  }
142}
143
144
145void do_bench(int nthreads, unsigned int size, int elsize, int rshift) {
146  void *src, *srccpy;
147  void **dest[NCHUNKS], *dest2;
148  int nbytes = 0, cbytes = 0;
149  size_t i, j;
150  struct timeval last, current;
151  float tmemcpy, tshuf, tunshuf;
152  int clevel, doshuffle=1;
153  unsigned char *orig, *round;
154
155  blosc_set_nthreads(nthreads);
156
157  /* Initialize buffers */
158  src = malloc(size);
159  srccpy = malloc(size);
160  dest2 = malloc(size);
161  /* zero src to initialize byte on it, and not only multiples of 4 */
162  memset(src, 0, size);
163  init_buffer(src, size, rshift);
164  memcpy(srccpy, src, size);
165  for (j = 0; j < nchunks; j++) {
166    /* 16 additional bytes should be enough for encoding everything */
167    dest[j] = malloc(size+16);
168  }
169
170  /* Warm destination memory (memcpy() will go a bit faster later on) */
171  for (j = 0; j < nchunks; j++) {
172    memcpy(dest[j], src, size);
173  }
174
175  printf("--> %d, %d, %d, %d\n", nthreads, size, elsize, rshift);
176  printf("********************** Run info ******************************\n");
177  printf("Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
178  printf("Using synthetic data with %d significant bits (out of 32)\n", rshift);
179  printf("Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
180  printf("Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
181  printf("Number of threads: %d\n", nthreads);
182  printf("********************** Running benchmarks *********************\n");
183
184  gettimeofday(&last, NULL);
185  for (i = 0; i < niter; i++) {
186    for (j = 0; j < nchunks; j++) {
187      memcpy(dest[j], src, size);
188    }
189  }
190  gettimeofday(&current, NULL);
191  tmemcpy = get_usec_chunk(last, current);
192  printf("memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
193         tmemcpy, size/(tmemcpy*MB/1e6));
194
195  gettimeofday(&last, NULL);
196  for (i = 0; i < niter; i++) {
197    for (j = 0; j < nchunks; j++) {
198      memcpy(dest2, dest[j], size);
199    }
200  }
201  gettimeofday(&current, NULL);
202  tmemcpy = get_usec_chunk(last, current);
203  printf("memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
204         tmemcpy, size/(tmemcpy*MB/1e6));
205
206  for (clevel=1; clevel<10; clevel++) {
207
208    printf("Compression level: %d\n", clevel);
209
210    gettimeofday(&last, NULL);
211    for (i = 0; i < niter; i++) {
212      for (j = 0; j < nchunks; j++) {
213        cbytes = blosc_compress(clevel, doshuffle, elsize, size, src,
214                                dest[j], size);
215      }
216    }
217    gettimeofday(&current, NULL);
218    tshuf = get_usec_chunk(last, current);
219    printf("comp(write):\t %6.1f us, %.1f MB/s\t  ",
220           tshuf, size/(tshuf*MB/1e6));
221    printf("Final bytes: %d  ", cbytes);
222    if (cbytes > 0) {
223      printf("Ratio: %3.2f", size/(float)cbytes);
224    }
225    printf("\n");
226
227    /* Compressor was unable to compress.  Copy the buffer manually. */
228    if (cbytes == 0) {
229      for (j = 0; j < nchunks; j++) {
230        memcpy(dest[j], src, size);
231      }
232    }
233
234    gettimeofday(&last, NULL);
235    for (i = 0; i < niter; i++) {
236      for (j = 0; j < nchunks; j++) {
237        if (cbytes == 0) {
238          memcpy(dest2, dest[j], size);
239          nbytes = size;
240        }
241        else {
242          nbytes = blosc_decompress(dest[j], dest2, size);
243        }
244      }
245    }
246    gettimeofday(&current, NULL);
247    tunshuf = get_usec_chunk(last, current);
248    printf("decomp(read):\t %6.1f us, %.1f MB/s\t  ",
249           tunshuf, nbytes/(tunshuf*MB/1e6));
250    if (nbytes < 0) {
251      printf("FAILED.  Error code: %d\n", nbytes);
252    }
253    /* printf("Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
254
255    /* Check if data has had a good roundtrip */
256    orig = (unsigned char *)srccpy;
257    round = (unsigned char *)dest2;
258    for(i = 0; i<size; ++i){
259      if (orig[i] != round[i]) {
260        printf("\nError: Original data and round-trip do not match in pos %d\n",
261               (int)i);
262        printf("Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
263        break;
264      }
265    }
266
267    if (i == size) printf("OK\n");
268
269  } /* End clevel loop */
270
271
272  /* To compute the totalsize, we should take into account the 9
273     compression levels */
274  totalsize += (size * nchunks * niter * 9.);
275
276  free(src); free(srccpy); free(dest2);
277  for (i = 0; i < nchunks; i++) {
278    free(dest[i]);
279  }
280
281}
282
283
284/* Compute a sensible value for nchunks */
285int get_nchunks(int size_, int ws) {
286  int nchunks;
287
288  nchunks = ws / size_;
289  if (nchunks > NCHUNKS) nchunks = NCHUNKS;
290  if (nchunks < 1) nchunks = 1;
291  return nchunks;
292}
293
294
295int main(int argc, char *argv[]) {
296  int single = 1;
297  int suite = 0;
298  int hard_suite = 0;
299  int extreme_suite = 0;
300  int debug_suite = 0;
301  int nthreads = 1;                     /* The number of threads */
302  int size = 2*MB;                      /* Buffer size */
303  int elsize = 8;                       /* Datatype size */
304  int rshift = 19;                      /* Significant bits */
305  int workingset = 256*MB;              /* The maximum allocated memory */
306  int nthreads_, size_, elsize_, rshift_, i;
307  struct timeval last, current;
308  float totaltime;
309  char *usage = "Usage: bench ['single' | 'suite' | 'hardsuite' | 'extremesuite' | 'debugsuite'] [nthreads [bufsize(bytes) [typesize [sbits ]]]]";
310
311
312  if (argc == 1) {
313    printf("%s\n", usage);
314    exit(1);
315  }
316
317  if (strcmp(argv[1], "single") == 0) {
318    single = 1;
319  }
320  else if (strcmp(argv[1], "suite") == 0) {
321    suite = 1;
322  }
323  else if (strcmp(argv[1], "hardsuite") == 0) {
324    hard_suite = 1;
325    workingset = 64*MB;
326    /* Values here are ending points for loops */
327    nthreads = 2;
328    size = 8*MB;
329    elsize = 32;
330    rshift = 32;
331  }
332  else if (strcmp(argv[1], "extremesuite") == 0) {
333    extreme_suite = 1;
334    workingset = 32*MB;
335    niter = 1;
336    /* Values here are ending points for loops */
337    nthreads = 4;
338    size = 16*MB;
339    elsize = 32;
340    rshift = 32;
341  }
342  else if (strcmp(argv[1], "debugsuite") == 0) {
343    debug_suite = 1;
344    workingset = 32*MB;
345    niter = 1;
346    /* Warning: values here are starting points for loops.  This is
347       useful for debugging. */
348    nthreads = 1;
349    size = 16*KB;
350    elsize = 1;
351    rshift = 0;
352  }
353  else {
354    printf("%s\n", usage);
355    exit(1);
356  }
357
358  if (argc >= 3) {
359    nthreads = atoi(argv[2]);
360  }
361  if (argc >= 4) {
362    size = atoi(argv[3]);
363  }
364  if (argc >= 5) {
365    elsize = atoi(argv[4]);
366  }
367  if (argc >= 6) {
368    rshift = atoi(argv[5]);
369  }
370
371  if ((argc >= 7) || !(single || suite || hard_suite || extreme_suite)) {
372    printf("%s\n", usage);
373    exit(1);
374  }
375
376  nchunks = get_nchunks(size, workingset);
377  gettimeofday(&last, NULL);
378
379  if (suite) {
380    for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) {
381        do_bench(nthreads_, size, elsize, rshift);
382    }
383  }
384  else if (hard_suite) {
385    for (rshift_ = 0; rshift_ <= rshift; rshift_ += 5) {
386      for (elsize_ = 1; elsize_ <= elsize; elsize_ *= 2) {
387        /* The next loop is for getting sizes that are not power of 2 */
388        for (i = -elsize_; i <= elsize_; i += elsize_) {
389          for (size_ = 32*KB; size_ <= size; size_ *= 2) {
390            nchunks = get_nchunks(size_+i, workingset);
391            niter = 1;
392            for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
393              do_bench(nthreads_, size_+i, elsize_, rshift_);
394              gettimeofday(&current, NULL);
395              totaltime = getseconds(last, current);
396              printf("Elapsed time:\t %6.1f s.  Processed data: %.1f GB\n",
397                     totaltime, totalsize / GB);
398            }
399          }
400        }
401      }
402    }
403  }
404  else if (extreme_suite) {
405    for (rshift_ = 0; rshift_ <= rshift; rshift_++) {
406      for (elsize_ = 1; elsize_ <= elsize; elsize_++) {
407        /* The next loop is for getting sizes that are not power of 2 */
408        for (i = -elsize_*2; i <= elsize_*2; i += elsize_) {
409          for (size_ = 32*KB; size_ <= size; size_ *= 2) {
410            nchunks = get_nchunks(size_+i, workingset);
411            for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
412              do_bench(nthreads_, size_+i, elsize_, rshift_);
413              gettimeofday(&current, NULL);
414              totaltime = getseconds(last, current);
415              printf("Elapsed time:\t %6.1f s.  Processed data: %.1f GB\n",
416                     totaltime, totalsize / GB);
417            }
418          }
419        }
420      }
421    }
422  }
423  else if (debug_suite) {
424    for (rshift_ = rshift; rshift_ <= 32; rshift_++) {
425      for (elsize_ = elsize; elsize_ <= 32; elsize_++) {
426        /* The next loop is for getting sizes that are not power of 2 */
427        for (i = -elsize_*2; i <= elsize_*2; i += elsize_) {
428          for (size_ = size; size_ <= 16*MB; size_ *= 2) {
429            nchunks = get_nchunks(size_+i, workingset);
430            for (nthreads_ = nthreads; nthreads_ <= 4; nthreads_++) {
431              do_bench(nthreads_, size_+i, elsize_, rshift_);
432              gettimeofday(&current, NULL);
433              totaltime = getseconds(last, current);
434              printf("Elapsed time:\t %6.1f s.  Processed data: %.1f GB\n",
435                     totaltime, totalsize / GB);
436            }
437          }
438        }
439      }
440    }
441  }
442  /* Single mode */
443  else {
444    do_bench(nthreads, size, elsize, rshift);
445  }
446
447  /* Print out some statistics */
448  gettimeofday(&current, NULL);
449  totaltime = getseconds(last, current);
450  printf("\nRound-trip compr/decompr on %.1f GB\n", totalsize / GB);
451  printf("Elapsed time:\t %6.1f s, %.1f MB/s\n",
452         totaltime, totalsize*2*1.1/(MB*totaltime));
453
454  /* Free blosc resources */
455  blosc_free_resources();
456
457  return 0;
458}
Note: See TracBrowser for help on using the repository browser.