squashfs-tools + zstd

5 files changed, 545 insertions, 0 deletions

diff --git a/sys-fs/squashfs-tools/files/squashfs-tools-4.3-aligned-data.patch b/sys-fs/squashfs-tools/files/squashfs-tools-4.3-aligned-data.patch
new file mode 100644
index 0000000..024129f
--- /dev/null
+++ b/sys-fs/squashfs-tools/files/squashfs-tools-4.3-aligned-data.patch
@@ -0,0 +1,15 @@
+the mksquashfs code has an all_zeros func that does:
+	long *p = (long *) file_buffer->data;
+
+make sure we force the data field to be aligned so that still works.
+
+--- a/squashfs-tools/caches-queues-lists.h
++++ b/squashfs-tools/caches-queues-lists.h
+@@ -123,6 +123,7 @@ struct file_buffer {
+ 	char locked;
+ 	char wait_on_unlock;
+ 	char noD;
++	long _pad;
+ 	char data[0];
+ };
+ 
diff --git a/sys-fs/squashfs-tools/files/squashfs-tools-4.3-sysmacros.patch b/sys-fs/squashfs-tools/files/squashfs-tools-4.3-sysmacros.patch
new file mode 100644
index 0000000..1430b1a
--- /dev/null
+++ b/sys-fs/squashfs-tools/files/squashfs-tools-4.3-sysmacros.patch
@@ -0,0 +1,25 @@
+sys/types.h might not always include sys/sysmacros.h for major/minor/makedev
+
+--- a/squashfs-tools/mksquashfs.c
++++ b/squashfs-tools/mksquashfs.c
+@@ -59,6 +59,7 @@
+ #else
+ #include <endian.h>
+ #include <sys/sysinfo.h>
++#include <sys/sysmacros.h>
+ #endif
+ 
+ #include "squashfs_fs.h"
+--- a/squashfs-tools/unsquashfs.c
++++ b/squashfs-tools/unsquashfs.c
+@@ -38,6 +38,10 @@
+ #include <limits.h>
+ #include <ctype.h>
+ 
++#ifdef linux
++#include <sys/sysmacros.h>
++#endif
++
+ struct cache *fragment_cache, *data_cache;
+ struct queue *to_reader, *to_inflate, *to_writer, *from_writer;
+ pthread_t *thread, *inflator_thread;
diff --git a/sys-fs/squashfs-tools/files/squashfs-tools-Add-zstd-support.patch b/sys-fs/squashfs-tools/files/squashfs-tools-Add-zstd-support.patch
new file mode 100644
index 0000000..ca638f2
--- /dev/null
+++ b/sys-fs/squashfs-tools/files/squashfs-tools-Add-zstd-support.patch
@@ -0,0 +1,420 @@
+From 57a3cf95b276946559f9e044c7352c11303bb9c1 Mon Sep 17 00:00:00 2001
+From: Sean Purcell <me@seanp.xyz>
+Date: Thu, 3 Aug 2017 17:47:03 -0700
+Subject: [PATCH v6] squashfs-tools: Add zstd support
+
+This patch adds zstd support to squashfs-tools. It works with zstd
+versions >= 1.0.0. It was originally written by Sean Purcell.
+
+Signed-off-by: Sean Purcell <me@seanp.xyz>
+Signed-off-by: Nick Terrell <terrelln@fb.com>
+---
+v4 -> v5:
+- Fix patch documentation to reflect that Sean Purcell is the author
+- Don't strip trailing whitespace of unreleated code
+- Make zstd_display_options() static
+
+v5 -> v6:
+- Fix build instructions in Makefile
+
+ squashfs-tools/Makefile       |  20 ++++
+ squashfs-tools/compressor.c   |   8 ++
+ squashfs-tools/squashfs_fs.h  |   1 +
+ squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++
+ squashfs-tools/zstd_wrapper.h |  48 ++++++++
+ 5 files changed, 331 insertions(+)
+ create mode 100644 squashfs-tools/zstd_wrapper.c
+ create mode 100644 squashfs-tools/zstd_wrapper.h
+
+diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile
+index 52d2582..22fc559 100644
+--- a/squashfs-tools/Makefile
++++ b/squashfs-tools/Makefile
+@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1
+ #LZMA_SUPPORT = 1
+ #LZMA_DIR = ../../../../LZMA/lzma465
+
++
++########### Building ZSTD support ############
++#
++# The ZSTD library is supported
++# ZSTD homepage: http://zstd.net
++# ZSTD source repository: https://github.com/facebook/zstd
++#
++# To build using the ZSTD library - install the library and uncomment the
++# ZSTD_SUPPORT line below.
++#
++#ZSTD_SUPPORT = 1
++
+ ######## Specifying default compression ########
+ #
+ # The next line specifies which compression algorithm is used by default
+@@ -177,6 +189,14 @@ LIBS += -llz4
+ COMPRESSORS += lz4
+ endif
+
++ifeq ($(ZSTD_SUPPORT),1)
++CFLAGS += -DZSTD_SUPPORT
++MKSQUASHFS_OBJS += zstd_wrapper.o
++UNSQUASHFS_OBJS += zstd_wrapper.o
++LIBS += -lzstd
++COMPRESSORS += zstd
++endif
++
+ ifeq ($(XATTR_SUPPORT),1)
+ ifeq ($(XATTR_DEFAULT),1)
+ CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT
+diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c
+index 525e316..02b5e90 100644
+--- a/squashfs-tools/compressor.c
++++ b/squashfs-tools/compressor.c
+@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {
+ extern struct compressor xz_comp_ops;
+ #endif
+
++#ifndef ZSTD_SUPPORT
++static struct compressor zstd_comp_ops = {
++	ZSTD_COMPRESSION, "zstd"
++};
++#else
++extern struct compressor zstd_comp_ops;
++#endif
+
+ static struct compressor unknown_comp_ops = {
+ 	0, "unknown"
+@@ -77,6 +84,7 @@ struct compressor *compressor[] = {
+ 	&lzo_comp_ops,
+ 	&lz4_comp_ops,
+ 	&xz_comp_ops,
++	&zstd_comp_ops,
+ 	&unknown_comp_ops
+ };
+
+diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h
+index 791fe12..afca918 100644
+--- a/squashfs-tools/squashfs_fs.h
++++ b/squashfs-tools/squashfs_fs.h
+@@ -277,6 +277,7 @@ typedef long long		squashfs_inode;
+ #define LZO_COMPRESSION		3
+ #define XZ_COMPRESSION		4
+ #define LZ4_COMPRESSION		5
++#define ZSTD_COMPRESSION	6
+
+ struct squashfs_super_block {
+ 	unsigned int		s_magic;
+diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c
+new file mode 100644
+index 0000000..dcab75a
+--- /dev/null
++++ b/squashfs-tools/zstd_wrapper.c
+@@ -0,0 +1,254 @@
++/*
++ * Copyright (c) 2017
++ * Phillip Lougher <phillip@squashfs.org.uk>
++ *
++ * This program is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU General Public License
++ * as published by the Free Software Foundation; either version 2,
++ * or (at your option) any later version.
++ *
++ * This program is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
++ * GNU General Public License for more details.
++ *
++ * zstd_wrapper.c
++ *
++ * Support for ZSTD compression http://zstd.net
++ */
++
++#include <stdio.h>
++#include <string.h>
++#include <stdlib.h>
++#include <zstd.h>
++#include <zstd_errors.h>
++
++#include "squashfs_fs.h"
++#include "zstd_wrapper.h"
++#include "compressor.h"
++
++static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
++
++/*
++ * This function is called by the options parsing code in mksquashfs.c
++ * to parse any -X compressor option.
++ *
++ * This function returns:
++ *	>=0 (number of additional args parsed) on success
++ *	-1 if the option was unrecognised, or
++ *	-2 if the option was recognised, but otherwise bad in
++ *	   some way (e.g. invalid parameter)
++ *
++ * Note: this function sets internal compressor state, but does not
++ * pass back the results of the parsing other than success/failure.
++ * The zstd_dump_options() function is called later to get the options in
++ * a format suitable for writing to the filesystem.
++ */
++static int zstd_options(char *argv[], int argc)
++{
++	if (strcmp(argv[0], "-Xcompression-level") == 0) {
++		if (argc < 2) {
++			fprintf(stderr, "zstd: -Xcompression-level missing "
++				"compression level\n");
++			fprintf(stderr, "zstd: -Xcompression-level it should "
++				"be 1 <= n <= %d\n", ZSTD_maxCLevel());
++			goto failed;
++		}
++
++		compression_level = atoi(argv[1]);
++		if (compression_level < 1 ||
++		    compression_level > ZSTD_maxCLevel()) {
++			fprintf(stderr, "zstd: -Xcompression-level invalid, it "
++				"should be 1 <= n <= %d\n", ZSTD_maxCLevel());
++			goto failed;
++		}
++
++		return 1;
++	}
++
++	return -1;
++failed:
++	return -2;
++}
++
++/*
++ * This function is called by mksquashfs to dump the parsed
++ * compressor options in a format suitable for writing to the
++ * compressor options field in the filesystem (stored immediately
++ * after the superblock).
++ *
++ * This function returns a pointer to the compression options structure
++ * to be stored (and the size), or NULL if there are no compression
++ * options.
++ */
++static void *zstd_dump_options(int block_size, int *size)
++{
++	static struct zstd_comp_opts comp_opts;
++
++	/* don't return anything if the options are all default */
++	if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL)
++		return NULL;
++
++	comp_opts.compression_level = compression_level;
++
++	SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
++
++	*size = sizeof(comp_opts);
++	return &comp_opts;
++}
++
++/*
++ * This function is a helper specifically for the append mode of
++ * mksquashfs.  Its purpose is to set the internal compressor state
++ * to the stored compressor options in the passed compressor options
++ * structure.
++ *
++ * In effect this function sets up the compressor options
++ * to the same state they were when the filesystem was originally
++ * generated, this is to ensure on appending, the compressor uses
++ * the same compression options that were used to generate the
++ * original filesystem.
++ *
++ * Note, even if there are no compressor options, this function is still
++ * called with an empty compressor structure (size == 0), to explicitly
++ * set the default options, this is to ensure any user supplied
++ * -X options on the appending mksquashfs command line are over-ridden.
++ *
++ * This function returns 0 on sucessful extraction of options, and -1 on error.
++ */
++static int zstd_extract_options(int block_size, void *buffer, int size)
++{
++	struct zstd_comp_opts *comp_opts = buffer;
++
++	if (size == 0) {
++		/* Set default values */
++		compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
++		return 0;
++	}
++
++	/* we expect a comp_opts structure of sufficient size to be present */
++	if (size < sizeof(*comp_opts))
++		goto failed;
++
++	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
++
++	if (comp_opts->compression_level < 1 ||
++	    comp_opts->compression_level > ZSTD_maxCLevel()) {
++		fprintf(stderr, "zstd: bad compression level in compression "
++			"options structure\n");
++		goto failed;
++	}
++
++	compression_level = comp_opts->compression_level;
++
++	return 0;
++
++failed:
++	fprintf(stderr, "zstd: error reading stored compressor options from "
++		"filesystem!\n");
++
++	return -1;
++}
++
++static void zstd_display_options(void *buffer, int size)
++{
++	struct zstd_comp_opts *comp_opts = buffer;
++
++	/* we expect a comp_opts structure of sufficient size to be present */
++	if (size < sizeof(*comp_opts))
++		goto failed;
++
++	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
++
++	if (comp_opts->compression_level < 1 ||
++	    comp_opts->compression_level > ZSTD_maxCLevel()) {
++		fprintf(stderr, "zstd: bad compression level in compression "
++			"options structure\n");
++		goto failed;
++	}
++
++	printf("\tcompression-level %d\n", comp_opts->compression_level);
++
++	return;
++
++failed:
++	fprintf(stderr, "zstd: error reading stored compressor options from "
++		"filesystem!\n");
++}
++
++/*
++ * This function is called by mksquashfs to initialise the
++ * compressor, before compress() is called.
++ *
++ * This function returns 0 on success, and -1 on error.
++ */
++static int zstd_init(void **strm, int block_size, int datablock)
++{
++	ZSTD_CCtx *cctx = ZSTD_createCCtx();
++
++	if (!cctx) {
++		fprintf(stderr, "zstd: failed to allocate compression "
++			"context!\n");
++		return -1;
++	}
++
++	*strm = cctx;
++	return 0;
++}
++
++static int zstd_compress(void *strm, void *dest, void *src, int size,
++			 int block_size, int *error)
++{
++	const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,
++					     src, size, compression_level);
++
++	if (ZSTD_isError(res)) {
++		/* FIXME:
++		 * zstd does not expose stable error codes. The error enum may
++		 * change between versions. Until upstream zstd stablizes the
++		 * error codes, we have no way of knowing why the error occurs.
++		 * zstd shouldn't fail to compress any input unless there isn't
++		 * enough output space. We assume that is the cause and return
++		 * the special error code for not enough output space.
++		 */
++		return 0;
++	}
++
++	return (int)res;
++}
++
++static int zstd_uncompress(void *dest, void *src, int size, int outsize,
++			   int *error)
++{
++	const size_t res = ZSTD_decompress(dest, outsize, src, size);
++
++	if (ZSTD_isError(res)) {
++		fprintf(stderr, "\t%d %d\n", outsize, size);
++
++		*error = (int)ZSTD_getErrorCode(res);
++		return -1;
++	}
++
++	return (int)res;
++}
++
++static void zstd_usage(void)
++{
++	fprintf(stderr, "\t  -Xcompression-level <compression-level>\n");
++	fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default "
++		"%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL);
++}
++
++struct compressor zstd_comp_ops = {
++	.init = zstd_init,
++	.compress = zstd_compress,
++	.uncompress = zstd_uncompress,
++	.options = zstd_options,
++	.dump_options = zstd_dump_options,
++	.extract_options = zstd_extract_options,
++	.display_options = zstd_display_options,
++	.usage = zstd_usage,
++	.id = ZSTD_COMPRESSION,
++	.name = "zstd",
++	.supported = 1
++};
+diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h
+new file mode 100644
+index 0000000..4fbef0a
+--- /dev/null
++++ b/squashfs-tools/zstd_wrapper.h
+@@ -0,0 +1,48 @@
++#ifndef ZSTD_WRAPPER_H
++#define ZSTD_WRAPPER_H
++/*
++ * Squashfs
++ *
++ * Copyright (c) 2017
++ * Phillip Lougher <phillip@squashfs.org.uk>
++ *
++ * This program is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU General Public License
++ * as published by the Free Software Foundation; either version 2,
++ * or (at your option) any later version.
++ *
++ * This program is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
++ * GNU General Public License for more details.
++ *
++ * zstd_wrapper.h
++ *
++ */
++
++#ifndef linux
++#define __BYTE_ORDER BYTE_ORDER
++#define __BIG_ENDIAN BIG_ENDIAN
++#define __LITTLE_ENDIAN LITTLE_ENDIAN
++#else
++#include <endian.h>
++#endif
++
++#if __BYTE_ORDER == __BIG_ENDIAN
++extern unsigned int inswap_le16(unsigned short);
++extern unsigned int inswap_le32(unsigned int);
++
++#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
++	(s)->compression_level = inswap_le32((s)->compression_level); \
++}
++#else
++#define SQUASHFS_INSWAP_COMP_OPTS(s)
++#endif
++
++/* Default compression */
++#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15
++
++struct zstd_comp_opts {
++	int compression_level;
++};
++#endif
+--
+2.9.5
diff --git a/sys-fs/squashfs-tools/metadata.xml b/sys-fs/squashfs-tools/metadata.xml
new file mode 100644
index 0000000..9edcb3d
--- /dev/null
+++ b/sys-fs/squashfs-tools/metadata.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE pkgmetadata SYSTEM "http://www.gentoo.org/dtd/metadata.dtd">
+<pkgmetadata>
+  <maintainer type="project">
+    <email>livecd@gentoo.org</email>
+    <name>Gentoo LiveCD Project</name>
+  </maintainer>
+  <use>
+    <flag name="xz">Enable support for XZ ("LZMA2") compression using <pkg>app-arch/xz-utils</pkg></flag>
+    <flag name="lz4">Enable support for LZ4 compression using <pkg>app-arch/lz4</pkg></flag>
+  </use>
+  <upstream>
+    <remote-id type="sourceforge">squashfs</remote-id>
+  </upstream>
+</pkgmetadata>
diff --git a/sys-fs/squashfs-tools/squashfs-tools-scm.ebuild b/sys-fs/squashfs-tools/squashfs-tools-scm.ebuild
new file mode 100644
index 0000000..f762a44
--- /dev/null
+++ b/sys-fs/squashfs-tools/squashfs-tools-scm.ebuild
@@ -0,0 +1,70 @@
+# Copyright 1999-2016 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=6
+
+inherit eutils git-r3 toolchain-funcs flag-o-matic
+
+DEB_VER="3"
+
+DESCRIPTION="Tool for creating compressed filesystem type squashfs"
+HOMEPAGE="http://squashfs.sourceforge.net"
+EGIT_REPO_URI="
+	https://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git
+"
+
+LICENSE="GPL-2"
+SLOT="0"
+KEYWORDS=""
+IUSE="debug lz4 lzma lzo static xattr +xz +zstd"
+
+LIB_DEPEND="sys-libs/zlib[static-libs(+)]
+	!xz? ( !lzo? ( sys-libs/zlib[static-libs(+)] ) )
+	lz4? ( app-arch/lz4[static-libs(+)] )
+	lzma? ( app-arch/xz-utils[static-libs(+)] )
+	lzo? ( dev-libs/lzo[static-libs(+)] )
+	xattr? ( sys-apps/attr[static-libs(+)] )
+	xz? ( app-arch/xz-utils[static-libs(+)] )
+	zstd? ( app-arch/zstd[static-libs(+)] )
+	"
+RDEPEND="!static? ( ${LIB_DEPEND//\[static-libs(+)]} )"
+DEPEND="${RDEPEND}
+	static? ( ${LIB_DEPEND} )"
+
+PATCHES=(
+	"${FILESDIR}"/${PN}-4.3-sysmacros.patch
+	"${FILESDIR}"/${PN}-4.3-aligned-data.patch
+	"${FILESDIR}"/${PN}-Add-zstd-support.patch
+)
+
+use10() { usex $1 1 0 ; }
+
+src_configure() {
+	cd "${WORKDIR}"/${P}/${PN} || die
+
+	# set up make command line variables in EMAKE_SQUASHFS_CONF
+	EMAKE_SQUASHFS_CONF=(
+		LZMA_XZ_SUPPORT=$(use10 lzma)
+		LZO_SUPPORT=$(use10 lzo)
+		LZ4_SUPPORT=$(use10 lz4)
+		XATTR_SUPPORT=$(use10 xattr)
+		XZ_SUPPORT=$(use10 xz)
+		ZSTD_SUPPORT=$(use10 zstd)
+	)
+
+	tc-export CC
+	use debug && append-cppflags -DSQUASHFS_TRACE
+	use static && append-ldflags -static
+}
+
+src_compile() {
+	cd "${WORKDIR}"/${P}/${PN} || die
+	emake "${EMAKE_SQUASHFS_CONF[@]}"
+}
+
+src_install() {
+	cd "${WORKDIR}"/${P}/${PN} || die
+	dobin mksquashfs unsquashfs
+	cd .. || die
+	dodoc CHANGES README RELEASE-README RELEASE-READMEs/*
+}