Added sharedCompression library

This commit is contained in:
Anonymous
2014-01-14 02:36:20 -07:00
parent a4f2c06f77
commit c7cf1b9faf
21 changed files with 1837 additions and 0 deletions
+1
View File
@@ -1,4 +1,5 @@
add_subdirectory(sharedCompression)
add_subdirectory(sharedDebug)
add_subdirectory(sharedFile)
add_subdirectory(sharedFoundation)
@@ -0,0 +1,11 @@
cmake_minimum_required(VERSION 2.8)
project(sharedCompression)
if(WIN32)
add_definitions(/D_CRT_SECURE_NO_WARNINGS)
endif()
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/public)
add_subdirectory(src)
@@ -0,0 +1 @@
#include "../../src/shared/BitStream.h"
@@ -0,0 +1 @@
#include "../../src/shared/Compressor.h"
@@ -0,0 +1 @@
#include "../../src/shared/FirstSharedCompression.h"
@@ -0,0 +1 @@
#include "../../src/shared/Lz77.h"
@@ -0,0 +1 @@
#include "../../src/shared/SetupSharedCompression.h"
@@ -0,0 +1 @@
#include "../../src/shared/ZlibCompressor.h"
@@ -0,0 +1,38 @@
set(SHARED_SOURCES
shared/BitStream.cpp
shared/BitStream.h
shared/Compressor.cpp
shared/Compressor.h
shared/FirstSharedCompression.h
shared/Lz77.cpp
shared/Lz77.h
shared/SetupSharedCompression.cpp
shared/SetupSharedCompression.h
shared/ZlibCompressor.cpp
shared/ZlibCompressor.h
)
if(WIN32)
set(PLATFORM_SOURCES
win32/FirstSharedCompression.cpp
)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/win32)
else()
set(PLATFORM_SOURCES "")
endif()
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/shared
${SWG_ENGINE_SOURCE_DIR}/shared/library/sharedDebug/include/public
${SWG_ENGINE_SOURCE_DIR}/shared/library/sharedFoundation/include/public
${SWG_ENGINE_SOURCE_DIR}/shared/library/sharedFoundationTypes/include/public
${SWG_ENGINE_SOURCE_DIR}/shared/library/sharedMemoryManager/include/public
${SWG_ENGINE_SOURCE_DIR}/shared/library/sharedSynchronization/include/public
)
add_library(sharedCompression STATIC
${SHARED_SOURCES}
${PLATFORM_SOURCES}
)
@@ -0,0 +1,527 @@
// ======================================================================
//
// BitStream.cpp
// ala diaz
//
// copyright 1999 Bootprint Entertainment
//
// ======================================================================
#include "sharedCompression/FirstSharedCompression.h"
#include "sharedCompression/BitStream.h"
const uint32 BitStream::INITIAL_MASK_VALUE = 0x80;
// ======================================================================
// Construct a BitStream
//
// Remarks:
//
// Initializes the components and sets the stream state.
BitStream::BitStream(bool inputStream)
: isInput(inputStream),
mask(INITIAL_MASK_VALUE),
rack(0)
{
}
// ----------------------------------------------------------------------
/**
* Destroy a BitStream.
*/
BitStream::~BitStream(void)
{
}
// ======================================================================
// Construct a BitBuffer
//
// Remarks:
//
// Initializes the buffer stream.
BitBuffer::BitBuffer(void *buffer, int size, bool inputStream)
: BitStream(inputStream),
base(static_cast<byte*>(NON_NULL(buffer))),
current(base),
bufSize(size)
{
DEBUG_FATAL(!buffer, ("buffer null"));
DEBUG_FATAL(!current, ("buffer null"));
}
// ----------------------------------------------------------------------
/**
* Destroy a BitBuffer.
*/
BitBuffer::~BitBuffer(void)
{
base = NULL;
current = NULL;
}
// ----------------------------------------------------------------------
int BitBuffer::getOffset(void) const
{
return current - base;
}
// ----------------------------------------------------------------------
/**
* Output bits to the stream.
*
* Outputs the lower count bits of the code to the stream.
* Count defaults to 1 bit.
*/
void BitBuffer::outputBits(uint32 code, uint32 count)
{
DEBUG_FATAL(isInput,("BitBuffer::outputBits called by input stream."));
uint32 codeMask = 1U << (count - 1);
while (codeMask)
{
if (codeMask & code)
{
rack |= mask;
}
mask >>= 1;
if (!mask)
{
DEBUG_FATAL(((current - base) * isizeof(rack)) >= bufSize, ("BitBuffer::outputBits attempting to write past end of buffer."));
*current++ = rack;
rack = 0;
mask = INITIAL_MASK_VALUE;
}
codeMask >>= 1;
}
}
// ----------------------------------------------------------------------
/**
* Flush the rack to the stream.
*
* If the current rack has not been written to, nothing happens.
* Otherwise, the rack is output to the stream and reset.
*/
void BitBuffer::outputRack(void)
{
if (mask != INITIAL_MASK_VALUE)
{
DEBUG_FATAL(((current - base) * isizeof(rack)) >= bufSize, ("BitBuffer::outputRack attempting to write past end of buffer."));
*current++ = rack;
rack = 0;
mask = INITIAL_MASK_VALUE;
}
}
// ----------------------------------------------------------------------
/**
* Input bits from the stream.
*
* Input count bits from the stream into the return value.
* Count defaults to 1 bit.
*
* @return The bits input from the stream.
*/
uint32 BitBuffer::inputBits(uint32 count)
{
DEBUG_FATAL(count > 32,("BitBuffer::inputBits can only input up to 32 bits at once, requested %d", count));
DEBUG_FATAL(!isInput,("BitBuffer::inputBits called by output stream."));
uint32 inputMask = 1U << (count - 1);
uint32 retval = 0;
while (inputMask)
{
if (mask == INITIAL_MASK_VALUE)
{
DEBUG_FATAL(((current - base) * isizeof(rack)) >= bufSize, ("BitBuffer::inputBits attempting to read past end of buffer."));
rack = *current++;
}
if (rack & mask)
{
retval |= inputMask;
}
inputMask >>= 1;
mask >>= 1;
if (!mask)
{
mask = INITIAL_MASK_VALUE;
}
}
return retval;
}
// ======================================================================
// Construct a BitFile
//
// Remarks:
//
// Initializes the file stream.
BitFile::BitFile(const char *fileName, bool inputStream)
: BitStream(inputStream),
hFile(INVALID_HANDLE_VALUE),
name(DuplicateString(fileName))
{
if (isInput)
{
hFile = CreateFile(fileName, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
}
else
{
// try to truncate it first
hFile = CreateFile(fileName, GENERIC_READ | GENERIC_WRITE, 0, NULL, TRUNCATE_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == INVALID_HANDLE_VALUE)
{
//if that fails, create it
hFile = CreateFile(fileName, GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
}
}
DEBUG_FATAL(hFile == INVALID_HANDLE_VALUE,("BitFile::BitFile - unable to open %s", fileName));
}
// ----------------------------------------------------------------------
/**
* Destroy a BitFile.
*
* Closes the file stream.
*/
BitFile::~BitFile(void)
{
if (hFile != INVALID_HANDLE_VALUE)
{
if (!CloseHandle(hFile))
{
DEBUG_FATAL(true,("BitFile::~BitFile - failed to close HANDLE for file %s - Error %d", name, GetLastError()));
}
}
hFile = NULL;
delete [] name;
}
// ----------------------------------------------------------------------
int BitFile::getOffset(void) const
{
return static_cast<int>(SetFilePointer(hFile, 0, NULL, FILE_CURRENT));
}
// ----------------------------------------------------------------------
/**
* Output bits to the stream.
*
* Outputs the lower count bits of the code to the stream.
* Count defaults to 1 bit.
*/
void BitFile::outputBits(uint32 code, uint32 count)
{
DEBUG_FATAL(isInput,("BitFile::outputBits called on input stream."));
uint32 codeMask = 1U << (count - 1);
while (codeMask)
{
if (codeMask & code)
{
rack |= mask;
}
mask >>= 1;
if (!mask)
{
uint32 bytesWritten;
if (!WriteFile(hFile, &rack, sizeof(rack), &bytesWritten, NULL))
{
DEBUG_FATAL(true,("BitFile::outputBits error %d.", GetLastError()));
}
rack = 0;
mask = INITIAL_MASK_VALUE;
}
codeMask >>= 1;
}
}
// ----------------------------------------------------------------------
/**
* Flush the rack to the stream.
*
* If the current rack has not been written to, nothing happens.
* Otherwise, the rack is output to the stream and reset.
*/
void BitFile::outputRack(void)
{
uint32 bytesWritten;
if (mask != INITIAL_MASK_VALUE)
{
if (!WriteFile(hFile, &rack, sizeof(rack), &bytesWritten, NULL))
{
DEBUG_FATAL(true,("BitFile::outputRack writing error %d.", GetLastError()));
}
rack = 0;
mask = INITIAL_MASK_VALUE;
}
}
// ----------------------------------------------------------------------
/**
* Input bits from the stream.
*
* Input count bits from the stream into the return value.
* Count defaults to 1 bit.
*/
uint32 BitFile::inputBits(uint32 count)
{
DEBUG_FATAL(count > 32,("BitFile::inputBits can only input up to 32 bits at once - requested %d", count));
DEBUG_FATAL(!isInput,("BitFile::inputBits called on output stream."));
uint32 inputMask = 1U << (count - 1);
uint32 retval = 0;
while (inputMask)
{
if (mask == INITIAL_MASK_VALUE )
{
uint32 bytesRead;
const BOOL result = ReadFile(hFile, &rack, sizeof(rack), &bytesRead, NULL);
UNREF(result);
DEBUG_FATAL(result && !bytesRead,("BitFile::inputBits hit EOF"));
}
if (rack & mask)
{
retval |= inputMask;
}
inputMask >>= 1;
mask >>= 1;
if (!mask)
{
mask = INITIAL_MASK_VALUE;
}
}
return retval;
}
// ======================================================================
// ======================================================================
// ======================================================================
// ======================================================================
// Construct a ByteStream
//
// Remarks:
//
// Initializes the stream state.
ByteStream::ByteStream(bool inputStream)
: isInput(inputStream)
{
}
// ----------------------------------------------------------------------
/**
* Destroy a ByteStream.
*/
ByteStream::~ByteStream(void)
{
}
// ======================================================================
// Construct a ByteBuffer
//
// Remarks:
//
// Initializes the buffer stream.
ByteBuffer::ByteBuffer(void *buffer, int size, bool inputStream)
: ByteStream(inputStream),
base(static_cast<byte*>(NON_NULL(buffer))),
current(base),
bufSize(size)
{
}
// ----------------------------------------------------------------------
/**
* Destroy a ByteBuffer.
*/
ByteBuffer::~ByteBuffer(void)
{
base = NULL;
current = NULL;
}
// ----------------------------------------------------------------------
/**
* Return the offset of the current position of the buffer.
*/
int ByteBuffer::getOffset(void) const
{
return current - base;
}
// ----------------------------------------------------------------------
/**
* Output a byte to the stream.
*/
void ByteBuffer::output(byte b)
{
DEBUG_FATAL((current - base) >= bufSize,("ByteBuffer::output attempting to write past end of buffer."));
DEBUG_FATAL(isInput,("ByteBuffer::output called on input stream"));
*current++ = b;
}
// ----------------------------------------------------------------------
/**
* Input a byte from the stream.
*
* Stores the next byte in the stream in b. Check the return value
* to make sure the stream is still valid.
*
* @return Returns true on end-of-stream, otherwise false.
*/
bool ByteBuffer::input(byte *b)
{
DEBUG_FATAL(!b, ("null pointer"));
DEBUG_FATAL(!isInput,("ByteBuffer::input called on output stream"));
if ((current - base) >= bufSize)
return true;
*b = *current++;
return false;
}
// ======================================================================
// Construct a ByteFile
//
// Remarks:
//
// Initializes the file stream.
ByteFile::ByteFile(const char *fileName, bool inputStream)
: ByteStream(inputStream),
hFile(INVALID_HANDLE_VALUE),
name(DuplicateString(fileName))
{
if (isInput)
{
hFile = CreateFile(fileName, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
}
else
{
hFile = CreateFile(fileName, GENERIC_READ | GENERIC_WRITE, 0, NULL, TRUNCATE_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == INVALID_HANDLE_VALUE)
{
hFile = CreateFile(fileName, GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
}
}
DEBUG_FATAL(hFile == INVALID_HANDLE_VALUE,("BitFile::BitFile - unable to open %s", fileName));
}
// ----------------------------------------------------------------------
/**
* Destroy a ByteFile.
*
* Closes the file stream.
*/
ByteFile::~ByteFile(void)
{
if (hFile != INVALID_HANDLE_VALUE && !CloseHandle(hFile))
DEBUG_FATAL(true,("ByteFile::~ByteFile - Unable to close HANDLE for file %s - Error %d", name, GetLastError()));
hFile = NULL;
delete [] name;
}
// ----------------------------------------------------------------------
int ByteFile::getOffset(void) const
{
return static_cast<int>(SetFilePointer(hFile, 0, NULL, FILE_CURRENT));
}
// ----------------------------------------------------------------------
/**
* Output a byte to the stream.
*/
void ByteFile::output(byte b)
{
DEBUG_FATAL(isInput,("ByteFile::output called on input stream."));
byte out = b;
uint32 bytesWritten;
if (!WriteFile(hFile, &out, sizeof(byte), &bytesWritten, NULL))
{
DEBUG_FATAL(true,("ByteFile::output error %d.", GetLastError()));
}
}
// ----------------------------------------------------------------------
/**
* Input a byte from the stream.
*
* Stores the next byte in the stream in b. Check the return value
* to make sure the stream is still valid.
*
* @return Returns true on end-of-stream, otherwise false.
*/
bool ByteFile::input(byte *b)
{
DEBUG_FATAL(!isInput,("ByteFile::input called on output stream."));
uint32 bytesRead;
BOOL result = ReadFile(hFile, b, sizeof(byte), &bytesRead, NULL);
// check for EOS
return (result && !bytesRead);
}
// ======================================================================
@@ -0,0 +1,192 @@
// =====================================================================
//
// BitStream.h
//
// ala diaz
//
// copyright 1999 Bootprint Entertainment
//
// =====================================================================
#ifndef BIT_STREAM_H
#define BIT_STREAM_H
// ======================================================================
// Base class for an i/o stream of bits.
class BitStream
{
private:
BitStream(void);
BitStream(const BitStream &);
BitStream &operator =(const BitStream &);
protected:
static const uint32 INITIAL_MASK_VALUE;
bool isInput;
byte mask;
byte rack;
public:
explicit BitStream(bool inputStream);
virtual ~BitStream(void);
virtual int getOffset(void) const = 0;
virtual void outputBits(uint32 code, uint32 count) = 0;
virtual void outputRack(void) = 0;
virtual uint32 inputBits(uint32 count) = 0;
};
// ======================================================================
// A buffer that streams bits. The buffer is NOT created by this class,
// it must be created externally and the location and size passed to the
// constructor.
class BitBuffer : public BitStream
{
private:
byte *base;
byte *current;
int bufSize;
private:
BitBuffer(void);
BitBuffer(const BitBuffer &);
BitBuffer &operator =(const BitBuffer &);
public:
BitBuffer(void *buffer, int size, bool inputStream);
virtual ~BitBuffer(void);
virtual int getOffset(void) const;
virtual void outputBits(uint32 code, uint32 count);
virtual void outputRack(void);
virtual uint32 inputBits(uint32 count);
};
// ======================================================================
// A file that streams bits. Mainly tools should cause the compressors to
// use BitFile.
class BitFile : public BitStream
{
private:
HANDLE hFile;
char *name;
private:
BitFile(void);
BitFile(const BitFile &);
BitFile &operator =(const BitFile &);
public:
BitFile(const char *fileName, bool inputStream);
virtual ~BitFile(void);
virtual int getOffset(void) const;
virtual void outputBits(uint32 code, uint32 count);
virtual void outputRack(void);
virtual uint32 inputBits(uint32 count);
};
// ======================================================================
// ======================================================================
// Base class for an i/o stream of bytes.
class ByteStream
{
private:
ByteStream(void);
ByteStream(const ByteStream &);
ByteStream &operator =(const ByteStream &);
protected:
bool isInput;
public:
explicit ByteStream(bool inputStream);
virtual ~ByteStream(void);
virtual int getOffset(void) const = 0;
virtual void output(byte b) = 0;
virtual bool input(byte *b) = 0;
};
// ======================================================================
// A buffer that streams bytes. The buffer is NOT created by this class,
// it must be created externally and the location and size passed to the
// constructor.
class ByteBuffer : public ByteStream
{
private:
byte *base;
byte *current;
int bufSize;
private:
ByteBuffer(void);
ByteBuffer(const ByteBuffer &);
ByteBuffer &operator =(const ByteBuffer &);
public:
ByteBuffer(void *buffer, int size, bool inputStream);
virtual ~ByteBuffer(void);
virtual int getOffset(void) const;
virtual void output(byte b);
virtual bool input(byte *b);
};
// ======================================================================
// A file that streams bytes. Mainly tools should cause the compressors
// to use ByteFile.
class ByteFile : public ByteStream
{
private:
HANDLE hFile;
char *name;
private:
ByteFile(void);
ByteFile(const ByteFile &);
ByteFile &operator =(const ByteFile &);
public:
ByteFile(const char *fileName, bool inputStream);
virtual ~ByteFile(void);
virtual int getOffset(void) const;
virtual void output(byte b);
virtual bool input(byte *b);
};
// ======================================================================
#endif
@@ -0,0 +1,34 @@
// =====================================================================
//
// Compressor.cpp
//
// ala diaz
//
// copyright 1999 Bootprint Entertainment
//
// =====================================================================
#include "sharedCompression/FirstSharedCompression.h"
#include "sharedCompression/Compressor.h"
#include "sharedCompression/BitStream.h"
// ----------------------------------------------------------------------
/**
* Construct a Compressor.
*/
Compressor::Compressor(void)
{
}
// ----------------------------------------------------------------------
/**
* Destroy a Compressor.
*/
Compressor::~Compressor(void)
{
}
// =====================================================================
@@ -0,0 +1,36 @@
// =====================================================================
//
// Compressor.h
//
// Portionc Copyright 1999 Bootprint Entertainment
// Portions Copyright 2003 Sony Online Entertainment
//
// =====================================================================
#ifndef INCLUDED_Compressor_H
#define INCLUDED_Compressor_H
// =====================================================================
class Compressor
{
public:
Compressor();
virtual ~Compressor();
virtual int compress(const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize) = 0;
virtual int expand (const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize) = 0;
virtual void compress(const char *inputFile, const char *outputFile) = 0;
virtual void expand (const char *inputFile, const char *outputFile) = 0;
private:
Compressor(const Compressor &);
Compressor &operator =(const Compressor &);
};
// =====================================================================
#endif
@@ -0,0 +1,17 @@
// ======================================================================
//
// FirstCompression.h
// copyright (c) 2001 Sony Online Entertainment
//
// ======================================================================
#ifndef INCLUDED_FirstCompression_H
#define INCLUDED_FirstCompression_H
// ======================================================================
#include "sharedFoundation/FirstSharedFoundation.h"
// ======================================================================
#endif
@@ -0,0 +1,483 @@
// =====================================================================
//
// Lz77.cpp
//
// ala diaz
//
// adapted from the LZSS algorithm in The Data Compression Book (Nelson & Gailly)
//
// Portions copyright 1999 Bootprint Entertainment
// Portions copyright 2001-2002 Sony Online Entertainment
// All Rights Reserved.
//
// =====================================================================
#include "sharedCompression/FirstSharedCompression.h"
#include "sharedCompression/Lz77.h"
#include "sharedCompression/BitStream.h"
// =====================================================================
// This is the LZSS module, which implements an Lz77 style compression
// algorithm. As iplemented here the default uses a 12 bit index into the
// sliding window, and a 4 bit length, which is adjusted to reflect phrase
// lengths of between 2 and 17 bytes.
// =====================================================================
const uint32 Lz77::byteEncodingCost = 9;
const uint32 Lz77::endOfStream = 0;
const uint32 Lz77::unused = 0;
// =====================================================================
// Construct a Lz77 compressor
//
// Remarks:
//
// Initializes all components of the compressor.
Lz77::Lz77(uint32 _indexBitCount, uint32 _lengthBitCount)
: Compressor(),
indexBitCount(_indexBitCount),
lengthBitCount(_lengthBitCount),
windowSize(1U << indexBitCount),
rawLookAheadSize(1U << lengthBitCount),
breakEven( (1 + indexBitCount + lengthBitCount) / byteEncodingCost ),
lookAheadSize(rawLookAheadSize + breakEven),
treeRoot(windowSize),
window(new byte[windowSize]),
tree(NULL)
{
}
// ----------------------------------------------------------------------
/**
* Destroy a Lz77 compressor.
*
* Frees memory used by compressor.
*/
Lz77::~Lz77(void)
{
delete [] window;
delete [] tree;
}
// ----------------------------------------------------------------------
/**
* Compress from buffer to buffer.
*
* Creates the buffer i/o streams and compresses.
*/
int Lz77::compress(const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize)
{
ByteBuffer in(const_cast<void *>(inputBuffer), inputSize, true);
BitBuffer out(outputBuffer, outputSize, false);
compress(in, out);
return out.getOffset();
}
// ----------------------------------------------------------------------
/**
* Compress from file to file.
*
* Creates the file i/o streams and compresses.
*/
void Lz77::compress(const char *inputName, const char *outputName)
{
ByteFile in(inputName, true);
BitFile out(outputName, false);
compress(in, out);
}
// ----------------------------------------------------------------------
/**
* Expand from buffer to buffer.
*
* Creates the buffer i/o streams and expands.
*/
int Lz77::expand(const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize)
{
BitBuffer in(const_cast<void *>(inputBuffer), inputSize, true);
ByteBuffer out(outputBuffer, outputSize, false);
expand(in, out);
return out.getOffset();
}
// ----------------------------------------------------------------------
/**
* Expand from file to file.
*
* Creates the file i/o streams and expands.
*/
void Lz77::expand(const char *inputName, const char *outputName)
{
BitFile in(inputName, true);
ByteFile out(outputName, false);
expand(in, out);
}
// ----------------------------------------------------------------------
/**
* Initialize the tree.
*
* Reset all nodes of the tree and set node r as the child of the root.
* The tree structure contains the binary tree of all the strings in
* the window sorted in order.
*/
void Lz77::initTree(uint32 r)
{
memset(tree, 0, static_cast<int>(sizeof(Node) * (windowSize + 1)));
tree[treeRoot].largeChild = r;
tree[r].parent = treeRoot;
tree[r].largeChild = unused;
tree[r].smallChild = unused;
}
// ----------------------------------------------------------------------
/**
* Initialize the text window.
*
* Zeroes out the text window.
*/
void Lz77::initWindow()
{
memset(window, 0, static_cast<int>(windowSize));
}
// ----------------------------------------------------------------------
/**
* Contract the tree.
*
* This routine is used when a node is being deleted. The link to
* oldNode is broken by pulling the newNode in to overlay the existing
* link.
*/
void Lz77::contractNode(uint32 oldNode, uint32 newNode)
{
tree[newNode].parent = tree[oldNode].parent;
if (tree[tree[oldNode].parent].largeChild == oldNode)
tree[tree[oldNode].parent].largeChild = newNode;
else
tree[tree[oldNode].parent].smallChild = newNode;
tree[oldNode].parent = unused;
}
// ----------------------------------------------------------------------
/**
* Replace a node in the tree with another.
*
* This routine is used when a node is being replaced by a node not
* previously in the tree.
*/
void Lz77::replaceNode(uint32 oldNode, uint32 newNode)
{
uint32 parent;
parent = tree[oldNode].parent;
if (tree[parent].smallChild == oldNode)
tree[parent].smallChild = newNode;
else
tree[parent].largeChild = newNode;
tree[newNode] = tree[oldNode];
tree[tree[newNode].smallChild].parent = newNode;
tree[tree[newNode].largeChild].parent = newNode;
tree[oldNode].parent = unused;
}
// ----------------------------------------------------------------------
/**
* Find the next smallest node.
*
* This routine is used to find the next smallest node after the node
* argument. It assumes that the node has a smaller child. We find
* the next smallest child by going to the smallChild node, then
* going to the end of the largeChild descendant chain.
*/
uint32 Lz77::findNextNode(uint32 node)
{
uint32 next;
next = tree[node].smallChild;
while (tree[next].largeChild != unused)
{
next = tree[next].largeChild;
}
return(next);
}
// ----------------------------------------------------------------------
/**
* Delete a string from the tree.
*
* This routine performs the classic binary tree deletion algorithm.
* If the node to be deleted has a null link in either direction, we
* just pull the non-null link up one to replace the existing link.
* If both links exist, we instead delete the next link in order, which
* is guaranteed to have a null link, then replace the node to be deleted
* with the next link.
*/
void Lz77::deleteString(uint32 p)
{
uint32 replacement;
if (tree[p].parent == unused)
return;
if ( tree[p].largeChild == unused)
{
contractNode(p, tree[p].smallChild);
}
else
if (tree[p].smallChild == unused)
{
contractNode(p, tree[p].largeChild);
}
else
{
replacement = findNextNode(p);
deleteString(replacement);
replaceNode(p, replacement);
}
}
// ----------------------------------------------------------------------
/**
* Add a string to the tree.
*
* This where most of the work done by the encoder takes place. This
* routine is responsible for adding the new node to the binary tree.
* It also has to find the best match among all the existing nodes in
* the tree, and return that to the calling routine. To make matters
* even more complicated, if the newNode has a duplicate in the tree,
* the oldNode is deleted, for reasons of efficiency.
*
* @return The length of the best match in the existing tree. The node where
* the best match begins if returned in matchPosition.
*/
uint32 Lz77::addString(uint32 newNode, uint32 *matchPosition)
{
uint32 i;
uint32 testNode;
uint32 matchLength;
uint32 *child;
int delta = 0;
DEBUG_FATAL(!matchPosition, ("matchPosition is NULL"));
if (newNode == endOfStream)
return 0;
testNode = tree[treeRoot].largeChild;
matchLength = 0;
for (;;)
{
for (i = 0 ; i < lookAheadSize ; i++)
{
delta = window[modWindow(newNode + i)] - window[modWindow(testNode + i)];
if (delta != 0)
break;
}
if (i >= matchLength)
{
matchLength = i;
*matchPosition = testNode;
if (matchLength >= lookAheadSize)
{
replaceNode(testNode, newNode);
return(matchLength);
}
}
if (delta >= 0)
child = &tree[testNode].largeChild;
else
child = &tree[testNode].smallChild;
if (*child == unused)
{
*child = newNode;
tree[newNode].parent = testNode;
tree[newNode].largeChild = unused;
tree[newNode].smallChild = unused;
return(matchLength);
}
testNode = *child;
}
}
// ----------------------------------------------------------------------
/**
* Compress a stream.
*
* This is the compression routine. It has to first load up the look
* ahead buffer, then go into the main compression loop. The main loop
* decides whether to output a single character or an index/length
* token that defines a phrase. Once the character or phrase has been
* sent out, another loop has to run. The second loop reads in new
* characters, deletes the strings that are overwritten by the new
* character, then adds the strings that are created by the new
* character.
*/
void Lz77::compress(ByteStream &in, BitStream &out)
{
uint32 i;
uint32 lookAheadBytes;
uint32 curPosition = 1;
uint32 replaceCount;
uint32 matchLength = 0;
uint32 matchPosition = 0;
byte c;
bool eos;
if (!tree)
tree = new Node[windowSize + 1];
initWindow();
initTree(curPosition);
for (i = 0; i < lookAheadSize; i++)
{
eos = in.input(&c);
if (eos)
{
break;
}
window[curPosition + i] = c;
}
lookAheadBytes = i;
while (lookAheadBytes > 0)
{
if (matchLength > lookAheadBytes)
{
matchLength = lookAheadBytes;
}
if (matchLength <= breakEven)
{
replaceCount = 1;
out.outputBits(1,1);
out.outputBits(window[curPosition],8);
}
else
{
out.outputBits(0,1);
out.outputBits(matchPosition, indexBitCount);
out.outputBits(matchLength - (breakEven + 1), lengthBitCount);
replaceCount = matchLength;
}
for (i = 0; i < replaceCount; i++)
{
deleteString(modWindow(curPosition + lookAheadSize));
eos = in.input(&c);
if (eos)
{
lookAheadBytes--;
}
else
{
window[modWindow(curPosition + lookAheadSize)] = c;
}
curPosition = modWindow(curPosition + 1);
if (lookAheadBytes)
{
matchLength = addString(curPosition, &matchPosition);
}
}
}
out.outputBits(0,1);
out.outputBits(endOfStream, indexBitCount);
out.outputRack();
}
// ----------------------------------------------------------------------
/**
* Expand a stream.
*
* This is the expansion routine for the LZSS algorithm. All it has
* to do is read in flag bits, decide whether to read in a character or
* a index/length pair, and take the appropriate action.
*/
void Lz77::expand(BitStream &in, ByteStream &out)
{
uint32 i;
byte c;
uint32 matchLength;
uint32 matchPosition;
uint32 curPosition = 1;
initWindow();
for(;;)
{
if (in.inputBits(1))
{
c = static_cast<byte>(in.inputBits(8));
out.output(c);
window[curPosition] = c;
curPosition = modWindow(curPosition + 1);
}
else
{
matchPosition = in.inputBits(indexBitCount);
if (matchPosition == endOfStream)
{
break;
}
matchLength = in.inputBits(lengthBitCount);
matchLength += breakEven;
for (i = 0; i <= matchLength; i++)
{
c = window[modWindow(matchPosition + i)];
out.output(c);
window[curPosition] = c;
curPosition = modWindow(curPosition + 1);
}
}
}
}
// =====================================================================
@@ -0,0 +1,120 @@
// =====================================================================
//
// Lz77.h
//
// Portions copyright 1999 Bootprint Entertainment
// Portions copyright 2002 Sony Online Entertainment
// All Rights Reserved.
//
// =====================================================================
#ifndef INCLUDED_Lz77_H
#define INCLUDED_Lz77_H
// =====================================================================
class ByteStream;
class BitStream;
#include "sharedCompression/Compressor.h"
// =====================================================================
class Lz77 : public Compressor
{
public:
explicit Lz77(uint32 _indexBitCount = 12, uint32 _lengthBitCount = 4);
virtual ~Lz77();
virtual int compress(const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize);
virtual int expand (const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize);
virtual void compress(const char *inputFile, const char *outputFile);
virtual void expand (const char *inputFile, const char *outputFile);
protected:
void compress(ByteStream &input, BitStream &output);
void expand(BitStream &input, ByteStream & output);
private:
struct Node
{
uint32 parent;
uint32 smallChild;
uint32 largeChild;
};
private:
uint32 modWindow(uint32 a);
void initTree(uint32 r);
void initWindow();
void contractNode(uint32 oldNode, uint32 newNode);
void replaceNode(uint32 oldNode, uint32 newNode);
uint32 findNextNode(uint32 node);
void deleteString(uint32 p);
uint32 addString(uint32 newNode, uint32 *matchPosition);
private:
// Cost in bits to store 1 byte
static const uint32 byteEncodingCost;
// Symbol signifying the end of the data stream
static const uint32 endOfStream;
// Index to an always-unused node in the tree
static const uint32 unused;
private:
// # of bits allocated to indices into the text window
const uint32 indexBitCount;
// # of bits allocated for the length of an encode phrase
const uint32 lengthBitCount;
// size of the text window
const uint32 windowSize;
// pre-adjusted size of look ahead buffer
const uint32 rawLookAheadSize;
// threshold where phrase encoding produces compression over storing
const uint32 breakEven;
// size of the look-ahead window
const uint32 lookAheadSize;
// permanent root of the tree belonging to no string
const uint32 treeRoot;
private:
byte *window;
Node *tree;
private:
Lz77(const Lz77&);
Lz77 &operator =(const Lz77&);
};
// =====================================================================
/**
* Protect window indices from overflowing and wrap them.
*
* @return The index after bounds wrapping.
*/
inline uint32 Lz77::modWindow(uint32 a)
{
return ((a) & (windowSize - 1));
}
// =====================================================================
#endif
@@ -0,0 +1,40 @@
// ======================================================================
//
// SetupSharedCompression.cpp
// Copyright 2003, Sony Online Entertainment Inc.
// All Rights Reserved.
//
// ======================================================================
#include "sharedCompression/FirstSharedCompression.h"
#include "sharedCompression/SetupSharedCompression.h"
#include "sharedCompression/ZlibCompressor.h"
#include "sharedDebug/InstallTimer.h"
// ======================================================================
SetupSharedCompression::Data::Data()
:
numberOfThreadsAccessingZlib(1)
{
}
// ======================================================================
void SetupSharedCompression::install()
{
InstallTimer const installTimer("SetupSharedCompression::install");
Data data;
install(data);
}
// ----------------------------------------------------------------------
void SetupSharedCompression::install(Data const &data)
{
ZlibCompressor::install(data.numberOfThreadsAccessingZlib);
}
// ======================================================================
@@ -0,0 +1,39 @@
// ======================================================================
//
// SetupSharedCompression.h
// Copyright 2003, Sony Online Entertainment Inc.
// All Rights Reserved.
//
// ======================================================================
#ifndef INCLUDED_SetupSharedCompression_H
#define INCLUDED_SetupSharedCompression_H
// ======================================================================
class SetupSharedCompression
{
public:
struct Data
{
int numberOfThreadsAccessingZlib;
Data();
};
public:
static void install();
static void install(Data const &data);
private:
SetupSharedCompression();
SetupSharedCompression(const SetupSharedCompression &);
SetupSharedCompression &operator =(const SetupSharedCompression &);
};
// ======================================================================
#endif
@@ -0,0 +1,242 @@
// ======================================================================
//
// ZlibCompressor.cpp
// Copyright 2003, Sony Online Entertainment Inc.
// All Rights Reserved.
//
// ======================================================================
#include "sharedCompression/FirstSharedCompression.h"
#include "sharedCompression/ZlibCompressor.h"
#include "sharedDebug/DebugFlags.h"
#include "sharedFoundation/ExitChain.h"
#include "sharedMemoryManager/MemoryManager.h"
#include "sharedSynchronization/Mutex.h"
#include "zlib.h"
#include <vector>
// ======================================================================
namespace ZlibCompressorNamespace
{
void remove();
voidpf allocateWrapper OF((voidpf opaque, uInt items, uInt size));
void freeWrapper OF((voidpf opaque, voidpf address));
int const cms_poolElementThreshold = 65536;
int ms_poolElementCount;
Mutex ms_mutex;
std::vector<void *> ms_memoryPool;
void * ms_memoryBottom;
void * ms_memoryTop;
}
using namespace ZlibCompressorNamespace;
// ======================================================================
voidpf ZlibCompressorNamespace::allocateWrapper(voidpf opaque, uInt items, uInt size)
{
UNREF(opaque);
void *result = 0;
#if 0
result = operator new(items * size);
#else
int totalSize = items * size;
if (totalSize > cms_poolElementThreshold)
{
result = operator new(totalSize);
}
else
{
ms_mutex.enter();
if (ms_memoryPool.empty())
{
result = operator new(totalSize);
}
else
{
result = ms_memoryPool.back();
ms_memoryPool.pop_back();
}
ms_mutex.leave();
}
#endif
return result;
}
// ----------------------------------------------------------------------
void ZlibCompressorNamespace::freeWrapper(voidpf opaque, voidpf address)
{
UNREF(opaque);
#if 0
operator delete(address);
#else
if (address < ms_memoryBottom || address >= ms_memoryTop)
operator delete(address);
else
{
ms_mutex.enter();
ms_memoryPool.push_back(address);
ms_mutex.leave();
}
#endif
}
// ======================================================================
void ZlibCompressor::install(int numberOfParallelThreads)
{
if (numberOfParallelThreads <= 1 || MemoryManager::getLimit() < 260)
ms_poolElementCount = 5;
else
if (numberOfParallelThreads <= 2 || MemoryManager::getLimit() < 375)
ms_poolElementCount = 10;
else
ms_poolElementCount = 15;
int const size = cms_poolElementThreshold * ms_poolElementCount;
ms_memoryBottom = operator new(size);
ms_memoryTop = reinterpret_cast<byte*>(ms_memoryBottom) + size;
for (int i = 0; i < ms_poolElementCount; ++i)
ms_memoryPool.push_back(reinterpret_cast<byte*>(ms_memoryBottom) + (i * cms_poolElementThreshold));
ExitChain::add(ZlibCompressorNamespace::remove, "ZlibCompressorNamespace::remove");
}
// ----------------------------------------------------------------------
void ZlibCompressorNamespace::remove()
{
ms_mutex.enter();
DEBUG_FATAL(static_cast<int>(ms_memoryPool.size()) != ms_poolElementCount, ("ZLibCompressor memory pool entries not all released"));
operator delete(ms_memoryBottom);
ms_memoryBottom = NULL;
ms_memoryTop = NULL;
ms_memoryPool.clear();
ms_mutex.leave();
}
// ======================================================================
ZlibCompressor::ZlibCompressor()
: Compressor()
{
}
// ----------------------------------------------------------------------
ZlibCompressor::~ZlibCompressor()
{
}
// ----------------------------------------------------------------------
int ZlibCompressor::compress(const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize)
{
z_stream z;
z.next_in = reinterpret_cast<Bytef *>(const_cast<void *>(inputBuffer));
z.avail_in = inputSize;
z.total_in = 0;
z.next_out = reinterpret_cast<Bytef *>(outputBuffer);
z.avail_out = outputSize;
z.total_out = 0;
z.msg = NULL;
z.state = NULL;
z.zalloc = allocateWrapper;
z.zfree = freeWrapper;
z.opaque = NULL;
z.data_type = Z_BINARY;
z.adler = 0;
z.reserved = 0;
if (deflateInit(&z, Z_DEFAULT_COMPRESSION) != Z_OK)
return -1;
if (deflate(&z, Z_FINISH) != Z_STREAM_END)
{
deflateEnd(&z);
return -1;
}
int const size = z.total_out;
if (deflateEnd(&z) != Z_OK)
return -1;
return size;
}
// ----------------------------------------------------------------------
int ZlibCompressor::expand(const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize)
{
z_stream z;
z.next_in = reinterpret_cast<Bytef *>(const_cast<void *>(inputBuffer));
z.avail_in = inputSize;
z.total_in = 0;
z.next_out = reinterpret_cast<Bytef *>(outputBuffer);
z.avail_out = outputSize;
z.total_out = 0;
z.msg = NULL;
z.state = NULL;
z.zalloc = allocateWrapper;
z.zfree = freeWrapper;
z.opaque = NULL;
z.data_type = Z_BINARY;
z.adler = 0;
z.reserved = 0;
if (inflateInit(&z) != Z_OK)
return -1;
if (inflate(&z, Z_FINISH) != Z_STREAM_END)
{
inflateEnd(&z);
return -1;
}
int const size = z.total_out;
if (inflateEnd(&z) != Z_OK)
return -1;
return size;
}
// ----------------------------------------------------------------------
void ZlibCompressor::compress(const char *inputFile, const char *outputFile)
{
UNREF(inputFile);
UNREF(outputFile);
}
// ----------------------------------------------------------------------
void ZlibCompressor::expand(const char *inputFile, const char *outputFile)
{
UNREF(inputFile);
UNREF(outputFile);
}
// ======================================================================
@@ -0,0 +1,43 @@
// ======================================================================
//
// ZlibCompressor.h
// Copyright 2003, Sony Online Entertainment Inc.
// All Rights Reserved.
//
// ======================================================================
#ifndef INCLUDED_ZlibCompressor_H
#define INCLUDED_ZlibCompressor_H
// ======================================================================
#include "sharedCompression/Compressor.h"
// ======================================================================
class ZlibCompressor : public Compressor
{
public:
static void install(int numberOfParallelThreads);
public:
ZlibCompressor();
virtual ~ZlibCompressor();
virtual int compress(const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize);
virtual int expand (const void *inputBuffer, int inputSize, void *outputBuffer, int outputSize);
virtual void compress(const char *inputFile, const char *outputFile);
virtual void expand (const char *inputFile, const char *outputFile);
private:
ZlibCompressor(const ZlibCompressor &);
ZlibCompressor &operator =(const ZlibCompressor &);
};
// ======================================================================
#endif
@@ -0,0 +1,8 @@
// ======================================================================
//
// FirstCompression.cpp
// copyright (c) 2001 Sony Online Entertainment
//
// ======================================================================
#include "sharedCompression/FirstSharedCompression.h"