zlib compatibility matrix

Article source code address: https://www.github.com/snowlyg/learns

Implementation of data compression/decompression between different languages

Normally, it is very simple to compress and decompress data using a programming language.

golang has the following 5 basic packages

The golang third-party package uses the c++ zlib package, which is mainly used this time.

chindeo/czlib

PHP has the following 6 compression and decompression formats

nodejs mainly uses zlib compression and decompression format

zlib

All three languages have zlib compression and decompression methods, so choosing zlib method for compression and decompression between different programming languages is more appropriate.

data.json file

{
    "fdafddsf":"e324r324e324e",
    "fd111afddsf":"e324r324e324e"
}

golang implements `zlib`

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path"

	"github.com/chindeo/czlib"
)

func main() {
	// 提取文件内容
	f, err := os.Open("./data.json")
	if err != nil {
		fmt.Printf("os open %v\n", err)
		return
	}

	// 压缩内容
	var b bytes.Buffer
	w, err := czlib.NewWriterLevel(&b, -1)
	if err != nil {
		fmt.Printf("flate new writer %v\n", err)
		return
	}
	defer w.Close()
	_, err = io.Copy(w, f)
	if err != nil {
		fmt.Printf("zlib new reader %v", err)
		return
	}
	w.Flush()

	// 将压缩后内容写入文件
	_, err = writeBytes("./data.zip", b.Bytes())
	if err != nil {
		fmt.Printf("WriteBytes %v\n", err)
		return
	}

	// 解压内容
	r, err := czlib.NewReader(&b)
	if err != nil {
		fmt.Printf("NewReader %v\n", err)
		return
	}
	defer r.Close()
	_, err = io.Copy(os.Stdout, r)
	if err != nil {
		fmt.Printf("io copy %v\n", err)
		return
	}
}

func writeBytes(filePath string, b []byte) (int, error) {
	os.MkdirAll(path.Dir(filePath), os.ModePerm)
	fw, err := os.Create(filePath)
	if err != nil {
		return 0, err
	}
	defer fw.Close()
	return fw.Write(b)
}

Execute go run main.go to get the data.zip file

nodejs implementation `zlib`

compress.js

const { createDeflate } = require('zlib');
const { pipeline } = require('stream');
const {
  createReadStream,
  createWriteStream
} = require('fs');

const gzip = createDeflate();
const source = createReadStream('data.json');
const destination = createWriteStream('data.zip');

pipeline(source, gzip, destination, (err) => {
  if (err) {
    console.error('发生错误:', err);
    process.exitCode = 1;
  }
});

-decompress.js

const { createInflate } = require('zlib');
const { pipeline } = require('stream');
const {
  createReadStream,
  createWriteStream
} = require('fs');


const gunzip = createInflate();
const sourceUnzip = createReadStream('data.zip');
const destinationUnzip =  createWriteStream('data.txt');

pipeline(sourceUnzip, gunzip, destinationUnzip, (err) => {
  if (err) {
    console.error('发生错误:', err);
    process.exitCode = 1;
  }
});

Execute node compress.js to get the data.zip file
Execute node decompress.js to decompress the data.zip file

php implements `zlib`

<?php
// Perform GZIP compression:
$compressed = file_get_contents("data.json");
$deflateContext = deflate_init(ZLIB_ENCODING_DEFLATE);
$compressed = deflate_add($deflateContext, $compressed, ZLIB_NO_FLUSH);
$compressed .= deflate_add($deflateContext, NULL, ZLIB_FINISH);

file_put_contents("data.zip",$compressed);

// Perform GZIP decompression:
$inflateContext = inflate_init(ZLIB_ENCODING_DEFLATE);
$uncompressed = inflate_add($inflateContext, $compressed, ZLIB_NO_FLUSH);
$uncompressed .= inflate_add($inflateContext, NULL, ZLIB_FINISH);
echo $uncompressed;
?>
?>

Execute php index.php to get the data.zip file

At present, the compression and decompression of data in each of the three languages have been implemented. Because the implementation is relatively simple, there is no code for cross-compression between languages and decompression of data. The specific source code reference is: learns/zlib ·

PHP decompresses GO compressed files

decompress_with_go_compress .php

<?php
// Perform GZIP compression:
$compressed = file_get_contents("../go/data.zip");
$inflateContext = inflate_init(ZLIB_ENCODING_DEFLATE);
$uncompressed = inflate_add($inflateContext, $compressed, ZLIB_NO_FLUSH);
$uncompressed .= inflate_add($inflateContext, NULL, ZLIB_FINISH);
file_put_contents("go_data.json",$uncompressed);
echo $uncompressed;
?>

PHP decompresses NODEJS compressed files

-decompress_with_nodejs_compress.php

<?php
// Perform GZIP compression:
$compressed = file_get_contents("../nodejs/data.zip");
$inflateContext = inflate_init(ZLIB_ENCODING_DEFLATE);
$uncompressed = inflate_add($inflateContext, $compressed, ZLIB_NO_FLUSH);
$uncompressed .= inflate_add($inflateContext, NULL, ZLIB_FINISH);
file_put_contents("go_data.json",$uncompressed);
echo $uncompressed;
?>

NODEJS decompresses PHP compressed files

decompress_with_php_compress.js

const { createInflate } = require('zlib');
const { pipeline } = require('stream');
const {
  createReadStream,
  createWriteStream
} = require('fs');


const gunzip = createInflate();
const sourceUnzip = createReadStream('../php/data.zip');
const destinationUnzip =  createWriteStream('php_data.txt');

pipeline(sourceUnzip, gunzip, destinationUnzip, (err) => {
  if (err) {
    console.error('发生错误:', err);
    process.exitCode = 1;
  }
});

NODEJS decompresses GO compressed files

decompress_with_go_compress.js

const { createInflate } = require('zlib');
const { pipeline } = require('stream');
const {
  createReadStream,
  createWriteStream
} = require('fs');


const gunzip = createInflate();
// console.log(gunzip)
const sourceUnzip = createReadStream('../go/data.zip');
const destinationUnzip =  createWriteStream('go_data.txt');
pipeline(sourceUnzip, gunzip, destinationUnzip, (err) => {
  if (err) {
    console.error('发生错误:', err);
    process.exitCode = 1;
  }
});

GO decompress PHP compressed files

package main

import (
	"fmt"
	"io"
	"os"

	"github.com/chindeo/czlib"
)

func main() {

	// 提取文件内容
	f, err := os.Open("../php/data.zip")
	if err != nil {
		fmt.Printf("os open %v\n", err)
		return
	}

	// 解压内容
	r, err := czlib.NewReader(f)
	if err != nil {
		fmt.Printf("NewReader %v\n", err)
		return
	}
	defer r.Close()
	_, err = io.Copy(os.Stdout, r)
	if err != nil {
		fmt.Printf("io copy %v\n", err)
		return
	}
}

GO decompress NODEJS compressed files

package main

import (
	"fmt"
	"io"
	"os"

	"github.com/chindeo/czlib"
)

func main() {

	// 提取文件内容
	f, err := os.Open("../nodejs/data.zip")
	if err != nil {
		fmt.Printf("os open %v\n", err)
		return
	}

	// 解压内容
	r, err := czlib.NewReader(f)
	if err != nil {
		fmt.Printf("NewReader %v\n", err)
		return
	}
	defer r.Close()
	_, err = io.Copy(os.Stdout, r)
	if err != nil {
		fmt.Printf("io copy %v\n", err)
		return
	}

}

This time go uses a third-party package chindeo/czlib. This package is modified with reference to an open source library. It mainly calls the zlib library of c++. Compared with several official compression libraries, this library can set more parameters and is more conveniently compatible with compression and decompression methods in other languages.

Especially when the compression and decompression methods used by other languages have been fixed for some reasons and are not easy to modify, it is often necessary to modify the corresponding parameters in other languages to match the corresponding compression settings