From aedb03fd86571708bc2bc1b2e5f64af07ae78661 Mon Sep 17 00:00:00 2001 From: Laurent Carlier Date: Sun, 28 Mar 2021 17:12:19 +0200 Subject: [PATCH] New 'gb.compress.zstd' component [GB.COMPRESS.ZSTD] * NEW: New 'gb.compress.zstd' component. --- Makefile.am | 1 + configure.ac | 1 + gb.compress.zstd/AUTHORS | 0 gb.compress.zstd/COPYING | 1 + gb.compress.zstd/ChangeLog | 0 gb.compress.zstd/INSTALL | 1 + gb.compress.zstd/Makefile.am | 3 + gb.compress.zstd/NEWS | 0 gb.compress.zstd/README | 0 gb.compress.zstd/acinclude.m4 | 1 + gb.compress.zstd/component.am | 1 + gb.compress.zstd/configure.ac | 23 + gb.compress.zstd/gambas.h | 1 + gb.compress.zstd/gb.compress.h | 1 + gb.compress.zstd/gb_common.h | 1 + gb.compress.zstd/m4 | 1 + gb.compress.zstd/reconf | 1 + gb.compress.zstd/src/Makefile.am | 10 + .../src/gb.compress.zstd.component | 4 + gb.compress.zstd/src/main.c | 393 ++++++++++++++++++ gb.compress.zstd/src/main.h | 48 +++ 21 files changed, 492 insertions(+) create mode 100644 gb.compress.zstd/AUTHORS create mode 120000 gb.compress.zstd/COPYING create mode 100644 gb.compress.zstd/ChangeLog create mode 120000 gb.compress.zstd/INSTALL create mode 100644 gb.compress.zstd/Makefile.am create mode 100644 gb.compress.zstd/NEWS create mode 100644 gb.compress.zstd/README create mode 120000 gb.compress.zstd/acinclude.m4 create mode 120000 gb.compress.zstd/component.am create mode 100644 gb.compress.zstd/configure.ac create mode 120000 gb.compress.zstd/gambas.h create mode 120000 gb.compress.zstd/gb.compress.h create mode 120000 gb.compress.zstd/gb_common.h create mode 120000 gb.compress.zstd/m4 create mode 120000 gb.compress.zstd/reconf create mode 100644 gb.compress.zstd/src/Makefile.am create mode 100644 gb.compress.zstd/src/gb.compress.zstd.component create mode 100644 gb.compress.zstd/src/main.c create mode 100644 gb.compress.zstd/src/main.h diff --git a/Makefile.am b/Makefile.am index 2f9188c8a..d1156b62c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -2,6 +2,7 @@ SUBDIRS = \ main \ @bzlib2_dir@ \ @zlib_dir@ \ + @zstd_dir@ \ @mysql_dir@ \ @odbc_dir@ \ @postgresql_dir@ \ diff --git a/configure.ac b/configure.ac index 132b1950b..91cb9d95c 100644 --- a/configure.ac +++ b/configure.ac @@ -14,6 +14,7 @@ AC_CONFIG_SUBDIRS(main) GB_CONFIG_SUBDIRS(bzlib2, gb.compress.bzlib2) GB_CONFIG_SUBDIRS(zlib, gb.compress.zlib) +GB_CONFIG_SUBDIRS(zstd, gb.compress.zstd) GB_CONFIG_SUBDIRS(mysql, gb.db.mysql) GB_CONFIG_SUBDIRS(odbc, gb.db.odbc) GB_CONFIG_SUBDIRS(postgresql, gb.db.postgresql) diff --git a/gb.compress.zstd/AUTHORS b/gb.compress.zstd/AUTHORS new file mode 100644 index 000000000..e69de29bb diff --git a/gb.compress.zstd/COPYING b/gb.compress.zstd/COPYING new file mode 120000 index 000000000..012065c85 --- /dev/null +++ b/gb.compress.zstd/COPYING @@ -0,0 +1 @@ +../COPYING \ No newline at end of file diff --git a/gb.compress.zstd/ChangeLog b/gb.compress.zstd/ChangeLog new file mode 100644 index 000000000..e69de29bb diff --git a/gb.compress.zstd/INSTALL b/gb.compress.zstd/INSTALL new file mode 120000 index 000000000..99d491b4f --- /dev/null +++ b/gb.compress.zstd/INSTALL @@ -0,0 +1 @@ +../INSTALL \ No newline at end of file diff --git a/gb.compress.zstd/Makefile.am b/gb.compress.zstd/Makefile.am new file mode 100644 index 000000000..0f297e179 --- /dev/null +++ b/gb.compress.zstd/Makefile.am @@ -0,0 +1,3 @@ +ACLOCAL_AMFLAGS = -I m4 --install +SUBDIRS = @ZSTD_DIR@ +EXTRA_DIST = reconf spec gambas.h gb*.h diff --git a/gb.compress.zstd/NEWS b/gb.compress.zstd/NEWS new file mode 100644 index 000000000..e69de29bb diff --git a/gb.compress.zstd/README b/gb.compress.zstd/README new file mode 100644 index 000000000..e69de29bb diff --git a/gb.compress.zstd/acinclude.m4 b/gb.compress.zstd/acinclude.m4 new file mode 120000 index 000000000..d84c32a31 --- /dev/null +++ b/gb.compress.zstd/acinclude.m4 @@ -0,0 +1 @@ +../acinclude.m4 \ No newline at end of file diff --git a/gb.compress.zstd/component.am b/gb.compress.zstd/component.am new file mode 120000 index 000000000..2f0eee34f --- /dev/null +++ b/gb.compress.zstd/component.am @@ -0,0 +1 @@ +../component.am \ No newline at end of file diff --git a/gb.compress.zstd/configure.ac b/gb.compress.zstd/configure.ac new file mode 100644 index 000000000..dc62f6c4c --- /dev/null +++ b/gb.compress.zstd/configure.ac @@ -0,0 +1,23 @@ +dnl ---- Initialization + +m4_include([../version.m4]) +AC_INIT(gambas3-gb-compress-zstd, GB_VERSION, GB_MAIL, [], GB_URL) +AC_CONFIG_MACRO_DIR([m4]) + +GB_INIT(gb.compress.zstd) +AC_PROG_LIBTOOL + +dnl ---- zstd compression driver + +GB_COMPONENT_PKG_CONFIG( + zstd, ZSTD, gb.compress.zstd, [src], + 'libzstd >= 1.4.0') + +dnl ---- Create makefiles + +AC_OUTPUT( \ +Makefile \ +src/Makefile \ +) + +GB_PRINT_MESSAGES diff --git a/gb.compress.zstd/gambas.h b/gb.compress.zstd/gambas.h new file mode 120000 index 000000000..03677ecd0 --- /dev/null +++ b/gb.compress.zstd/gambas.h @@ -0,0 +1 @@ +../main/share/gambas.h \ No newline at end of file diff --git a/gb.compress.zstd/gb.compress.h b/gb.compress.zstd/gb.compress.h new file mode 120000 index 000000000..7215a72a4 --- /dev/null +++ b/gb.compress.zstd/gb.compress.h @@ -0,0 +1 @@ +../main/lib/compress/gb.compress.h \ No newline at end of file diff --git a/gb.compress.zstd/gb_common.h b/gb.compress.zstd/gb_common.h new file mode 120000 index 000000000..707d79da6 --- /dev/null +++ b/gb.compress.zstd/gb_common.h @@ -0,0 +1 @@ +../main/share/gb_common.h \ No newline at end of file diff --git a/gb.compress.zstd/m4 b/gb.compress.zstd/m4 new file mode 120000 index 000000000..7d49a2a4b --- /dev/null +++ b/gb.compress.zstd/m4 @@ -0,0 +1 @@ +../m4 \ No newline at end of file diff --git a/gb.compress.zstd/reconf b/gb.compress.zstd/reconf new file mode 120000 index 000000000..48a376da6 --- /dev/null +++ b/gb.compress.zstd/reconf @@ -0,0 +1 @@ +../reconf \ No newline at end of file diff --git a/gb.compress.zstd/src/Makefile.am b/gb.compress.zstd/src/Makefile.am new file mode 100644 index 000000000..ae117b49c --- /dev/null +++ b/gb.compress.zstd/src/Makefile.am @@ -0,0 +1,10 @@ +COMPONENT = gb.compress.zstd +include $(top_srcdir)/component.am + +gblib_LTLIBRARIES = gb.compress.zstd.la + +gb_compress_zstd_la_LIBADD = @ZSTD_LIB@ +gb_compress_zstd_la_LDFLAGS = -module @LD_FLAGS@ @ZSTD_LDFLAGS@ +gb_compress_zstd_la_CFLAGS = $(AM_CFLAGS) @ZSTD_INC@ + +gb_compress_zstd_la_SOURCES = main.h main.c diff --git a/gb.compress.zstd/src/gb.compress.zstd.component b/gb.compress.zstd/src/gb.compress.zstd.component new file mode 100644 index 000000000..695a8a734 --- /dev/null +++ b/gb.compress.zstd/src/gb.compress.zstd.component @@ -0,0 +1,4 @@ +[Component] +Key=gb.compress.zstd +Author=Laurent Carlier +Require=gb.compress diff --git a/gb.compress.zstd/src/main.c b/gb.compress.zstd/src/main.c new file mode 100644 index 000000000..830205aeb --- /dev/null +++ b/gb.compress.zstd/src/main.c @@ -0,0 +1,393 @@ +/*************************************************************************** + + main.c + + (c) 2019-present Laurent Carlier + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. + +***************************************************************************/ + +#define __MAIN_C + +#include "gb_common.h" + +// Use 64 bits I/O +#if (__WORDSIZE == 64) && (!_LARGEFILE64_SOURCE) +#define _LARGEFILE64_SOURCE +#endif + +#include +#include + +#include "main.h" + +#define MODE_READ 0 +#define MODE_WRITE 1 + +GB_INTERFACE EXPORT GB; +COMPRESS_INTERFACE EXPORT COMPRESSION; + +static COMPRESS_DRIVER _driver; + +GB_STREAM_DESC ZSTDStream = { + .open = ZSTD_stream_open, + .close = ZSTD_stream_close, + .read = ZSTD_stream_read, + .write = ZSTD_stream_write, + .seek = ZSTD_stream_seek, + .tell = ZSTD_stream_tell, + .flush = ZSTD_stream_flush, + .eof = ZSTD_stream_eof, + .lof = ZSTD_stream_lof +}; + +typedef + struct + { + uint8_t mode; + } + handleInfo; + +typedef + struct { + GB_STREAM_BASE base; + handleInfo *info; + } + STREAM_COMPRESS; + + +// /***************************************************************************** +// +// The driver interface +// +// *****************************************************************************/ + +static int ZSTD_max_compression(void) +{ + return ZSTD_maxCLevel(); +} + +static int ZSTD_min_compression(void) +{ + return ZSTD_minCLevel(); +} + +static int ZSTD_default_compression(void) +{ + return ZSTD_CLEVEL_DEFAULT; +} + +/***************************************************************************** + + Compression + +*****************************************************************************/ + +static void ZSTD_c_String(char **target,unsigned int *lent,char *source,unsigned int len,int level) +{ + size_t retValue; + + *lent = ZSTD_compressBound(len); + GB.Alloc((void**)target, sizeof(char)*(*lent)); + + retValue = ZSTD_compress((void *)(*target), *lent, (const void *)source, len, level); + + if (ZSTD_isError(retValue)) + { + GB.Free((void**)target); + *lent = 0; + *target = NULL; + GB.Error("Unable to compress string: &1", ZSTD_getErrorName(retValue)); + return; + } + + *lent = (uint)retValue; +} + +static void ZSTD_c_File(char *source,char *target,int level) +{ + FILE *f_src, *f_dst; + void *buffIn, *buffOut; + + if ((f_src=fopen(source, "rb")) == NULL) + { + GB.Error("Unable to open file for reading"); + return; + } + + if ((f_dst=fopen(target, "wb")) == NULL) + { + fclose(f_src); + GB.Error("Unable to open file for writing"); + return; + } + + size_t const buffInSize = ZSTD_CStreamInSize(); + size_t const buffOutSize = ZSTD_CStreamOutSize(); + GB.Alloc(&buffIn, buffInSize); + GB.Alloc(&buffOut, buffOutSize); + + // TODO: it is recommended to allocate a context only once, and re-use it for each successive compression operation. + ZSTD_CCtx *cctx = ZSTD_createCCtx(); + + if (cctx == NULL) + { + GB.Error("Error while compressing file: ZSTD_createCCtx() failed!"); + goto error_cfile; + } + + // TODO: check errors ? + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + + ZSTD_EndDirective mode = ZSTD_e_continue; + while(!feof(f_src)) + { + size_t len = fread(buffIn, 1, buffInSize, f_src); + + if (len < buffInSize) + { + if (ferror(f_src)) + { + GB.Error("Error while reading data: &1", strerror(errno)); + goto error_cfile; + } + mode = ZSTD_e_end; + } + + if (len > 0) + { + ZSTD_inBuffer input = { buffIn, len, 0 }; + int finished; + do { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode); + + if (ZSTD_isError(remaining)) + { + GB.Error("Error while compressing file: &1", ZSTD_getErrorName(remaining)); + goto error_cfile; + } + if (fwrite(buffOut, 1, output.pos, f_dst) != output.pos) + { + GB.Error("Error while writing data: &1", strerror(errno)); + goto error_cfile; + } + + finished = (len < buffInSize) ? (remaining == 0) : (input.pos == input.size); + } while (!finished); + } + } + +error_cfile: + if (cctx != NULL) + ZSTD_freeCCtx(cctx); + GB.Free(&buffIn); + GB.Free(&buffOut); + fclose(f_src); + fclose(f_dst); + return; +} + + +static void ZSTD_c_Open(char *path,int level, STREAM_COMPRESS *stream) +{ + GB.Error("Not yet implemented!"); + return; +} + +/***************************************************************************** + + Uncompression + +*****************************************************************************/ + +static void ZSTD_u_String(char **target,unsigned int *lent,char *source,unsigned int len) +{ + *lent = ZSTD_getFrameContentSize((const void *) source, len); + size_t dSize = 0; + + if (*lent == ZSTD_CONTENTSIZE_UNKNOWN) + { + GB.Error("Size unknown - maybe needs streaming - not yet implemented!"); + return; + } + if (*lent == ZSTD_CONTENTSIZE_ERROR) + { + GB.Error("Not compressed by zstd!"); + return; + } + + GB.Alloc((void **) target, sizeof(char)*(*lent)); + dSize = ZSTD_decompress((void *)(*target), *lent, (const void *)source, len); + + if (ZSTD_isError(dSize)) + { + GB.Free((void**)target); + *lent=0; + *target=NULL; + GB.Error("Unable to uncompress string: &1", ZSTD_getErrorName(dSize)); + return; + } + + *lent = (uint)dSize; +} + +static void ZSTD_u_File(char *source,char *target) +{ + FILE *f_src, *f_dst; + void *buffIn, *buffOut; + + if ((f_src=fopen(source, "rb")) == NULL) + { + GB.Error("Unable to open file for reading"); + return; + } + + if ((f_dst=fopen(target, "wb")) == NULL) + { + fclose(f_src); + GB.Error("Unable to open file for writing"); + return; + } + + size_t const buffInSize = ZSTD_DStreamInSize(); + size_t const buffOutSize = ZSTD_DStreamOutSize(); + GB.Alloc(&buffIn, buffInSize); + GB.Alloc(&buffOut, buffOutSize); + + // TODO: it is recommended to allocate a context only once, and re-use it for each successive decompression operation. + ZSTD_DCtx *dctx = ZSTD_createDCtx(); + + if (dctx == NULL) + { + GB.Error("Error while decompressing file: ZSTD_createDCtx() failed!"); + goto error_ufile; + } + + while(!feof(f_src)) + { + size_t len = fread(buffIn, 1, buffInSize, f_src); + + if (len < buffInSize) + { + if (len == 0) + { + GB.Error("Error: input file is empty"); + goto error_ufile; + } + if (ferror(f_src)) + { + GB.Error("Error while reading data: &1", strerror(errno)); + goto error_ufile; + } + } + + if (len > 0) + { + ZSTD_inBuffer input = { buffIn, len, 0 }; + while (input.pos < input.size) + { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + size_t const ret = ZSTD_decompressStream(dctx, &output , &input); + + if (ZSTD_isError(ret)) + { + GB.Error("Error while decompressing file: &1", ZSTD_getErrorName(ret)); + goto error_ufile; + } + + if (fwrite(buffOut, 1, output.pos, f_dst) != output.pos) + { + GB.Error("Error while writing data: &1", strerror(errno)); + goto error_ufile; + } + } + } + } + +error_ufile: + if (dctx != NULL) + ZSTD_freeDCtx(dctx); + GB.Free(&buffIn); + GB.Free(&buffOut); + fclose(f_src); + fclose(f_dst); + return; +} + +static void ZSTD_u_Open(char *path, STREAM_COMPRESS *stream) +{ + GB.Error("Not yet implemented!"); + return; +} + +/************************************************************************* +Stream related stuff +**************************************************************************/ +/* TODO */ +static int ZSTD_stream_open(GB_STREAM *stream, const char *path, int mode, void *data) {return -1;} +static int ZSTD_stream_close(GB_STREAM *stream) {return -1;} +static int ZSTD_stream_read(GB_STREAM *stream, char *buffer, int len) {return -1;} +static int ZSTD_stream_write(GB_STREAM *stream, char *buffer, int len) {return -1;} +static int ZSTD_stream_seek(GB_STREAM *stream, int64_t offset, int whence) {return -1;} +static int ZSTD_stream_tell(GB_STREAM *stream, int64_t *npos) {return -1;} +static int ZSTD_stream_flush(GB_STREAM *stream) {return -1;} +static int ZSTD_stream_eof(GB_STREAM *stream) {return -1;} +static int ZSTD_stream_lof(GB_STREAM *stream, int64_t *len) {return -1;} +/* end of TODO */ + +static COMPRESS_DRIVER _driver = +{ + "zstd", + + (void*)ZSTD_max_compression, + (void*)ZSTD_min_compression, + (void*)ZSTD_default_compression, + + { + (void*)ZSTD_c_String, + (void*)ZSTD_c_File, + (void*)ZSTD_c_Open, + (void*)ZSTD_stream_close, + }, + + { + (void*)ZSTD_u_String, + (void*)ZSTD_u_File, + (void*)ZSTD_u_Open, + (void*)ZSTD_stream_close + } +}; + +/***************************************************************************** + + The component entry and exit functions. + +*****************************************************************************/ + +int EXPORT GB_INIT(void) +{ + GB.GetInterface("gb.compress", COMPRESS_INTERFACE_VERSION, &COMPRESSION); + COMPRESSION.Register(&_driver); + + return 0; +} + +void EXPORT GB_EXIT() +{ +} diff --git a/gb.compress.zstd/src/main.h b/gb.compress.zstd/src/main.h new file mode 100644 index 000000000..b2385c782 --- /dev/null +++ b/gb.compress.zstd/src/main.h @@ -0,0 +1,48 @@ +/*************************************************************************** + + main.h + + (c) 2019-present Laurent Carlier + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. + +***************************************************************************/ + +#ifndef __MAIN_H +#define __MAIN_H + +#include "gambas.h" +#include "gb_common.h" +#include "../gb.compress.h" +#include + +#ifndef __MAIN_C +extern GB_INTERFACE GB; +extern COMPRESS_INTERFACE COMPRESSION; +extern GB_STREAM_DESC ZSTDStream; +#endif + +static int ZSTD_stream_open(GB_STREAM *stream, const char *path, int mode, void *data); +static int ZSTD_stream_close(GB_STREAM *stream); +static int ZSTD_stream_read(GB_STREAM *stream, char *buffer, int len); +static int ZSTD_stream_write(GB_STREAM *stream, char *buffer, int len); +static int ZSTD_stream_seek(GB_STREAM *stream, int64_t offset, int whence); +static int ZSTD_stream_tell(GB_STREAM *stream, int64_t *npos); +static int ZSTD_stream_flush(GB_STREAM *stream); +static int ZSTD_stream_eof(GB_STREAM *stream); +static int ZSTD_stream_lof(GB_STREAM *stream, int64_t *len); + +#endif /* __MAIN_H */