From 1e32c161ab1c79c6d5554becbc17f00beeb607af Mon Sep 17 00:00:00 2001 From: Elour Date: Mon, 25 Apr 2022 17:44:06 -0700 Subject: [PATCH] initial commit this includes the TOCBuilder program --- .gitignore | 49 ++ .idea/.gitignore | 3 + .idea/artifacts/TOCBuilder_jar.xml | 10 + .idea/codeStyles/Project.xml | 22 + .idea/codeStyles/codeStyleConfig.xml | 5 + .idea/description.html | 1 + .idea/encodings.xml | 6 + .idea/libraries/ant_ant_jsch_1_6_5.xml | 10 + .idea/libraries/com_jcraft_jsch_0_1_52.xml | 10 + .idea/misc.xml | 9 + .idea/modules.xml | 8 + .idea/project-template.xml | 3 + .idea/uiDesigner.xml | 124 +++++ .idea/vcs.xml | 6 + LegendsUtils.iml | 29 ++ src/META-INF/MANIFEST.MF | 3 + src/io/elour/TOCBuilder.java | 345 ++++++++++++++ src/io/elour/TableOfContentsEntry.java | 93 ++++ src/io/elour/TocHeader.java | 139 ++++++ src/io/elour/TreeHeader.java | 59 +++ src/io/elour/TreeTOCEntry.java | 57 +++ src/io/elour/Utils.java | 517 +++++++++++++++++++++ 22 files changed, 1508 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/artifacts/TOCBuilder_jar.xml create mode 100644 .idea/codeStyles/Project.xml create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/description.html create mode 100644 .idea/encodings.xml create mode 100644 .idea/libraries/ant_ant_jsch_1_6_5.xml create mode 100644 .idea/libraries/com_jcraft_jsch_0_1_52.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/project-template.xml create mode 100644 .idea/uiDesigner.xml create mode 100644 .idea/vcs.xml create mode 100644 LegendsUtils.iml create mode 100644 src/META-INF/MANIFEST.MF create mode 100644 src/io/elour/TOCBuilder.java create mode 100644 src/io/elour/TableOfContentsEntry.java create mode 100644 src/io/elour/TocHeader.java create mode 100644 src/io/elour/TreeHeader.java create mode 100644 src/io/elour/TreeTOCEntry.java create mode 100644 src/io/elour/Utils.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0f5d17d --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +node_modules/ +dist/ + +# Compiled Java class files +*.class + +# Compiled Python bytecode +*.py[cod] + +# Log files +*.log + +# Package files +*.jar + +# Maven +target/ +dist/ + +# Unit test reports +TEST*.xml + +# Generated by MacOS +.DS_Store + +# Generated by Windows +Thumbs.db + +# Applications +*.app +*.exe +*.war + +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv + +#ignore out dir +out/ \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..eaf91e2 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/artifacts/TOCBuilder_jar.xml b/.idea/artifacts/TOCBuilder_jar.xml new file mode 100644 index 0000000..f0fa964 --- /dev/null +++ b/.idea/artifacts/TOCBuilder_jar.xml @@ -0,0 +1,10 @@ + + + $PROJECT_DIR$/out/artifacts/ + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..da6b07e --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,22 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..df5f35d --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/description.html b/.idea/description.html new file mode 100644 index 0000000..db5f129 --- /dev/null +++ b/.idea/description.html @@ -0,0 +1 @@ +Simple Java application that includes a class with main() method \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..ab2dc53 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/libraries/ant_ant_jsch_1_6_5.xml b/.idea/libraries/ant_ant_jsch_1_6_5.xml new file mode 100644 index 0000000..cb551cd --- /dev/null +++ b/.idea/libraries/ant_ant_jsch_1_6_5.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/com_jcraft_jsch_0_1_52.xml b/.idea/libraries/com_jcraft_jsch_0_1_52.xml new file mode 100644 index 0000000..402400c --- /dev/null +++ b/.idea/libraries/com_jcraft_jsch_0_1_52.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..c05ca30 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,9 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..c9460e3 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/project-template.xml b/.idea/project-template.xml new file mode 100644 index 0000000..d57a956 --- /dev/null +++ b/.idea/project-template.xml @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..b93ac08 --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..c8397c9 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/LegendsUtils.iml b/LegendsUtils.iml new file mode 100644 index 0000000..378ebee --- /dev/null +++ b/LegendsUtils.iml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/META-INF/MANIFEST.MF b/src/META-INF/MANIFEST.MF new file mode 100644 index 0000000..8f6516d --- /dev/null +++ b/src/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Main-Class: io.elour.TOCBuilder + diff --git a/src/io/elour/TOCBuilder.java b/src/io/elour/TOCBuilder.java new file mode 100644 index 0000000..7bb8d11 --- /dev/null +++ b/src/io/elour/TOCBuilder.java @@ -0,0 +1,345 @@ +package io.elour; + +import java.io.BufferedWriter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +import static io.elour.Utils.*; + +public class TOCBuilder { + + public static void main(String[] args) { + String outFilename = args[0]; + FileOutputStream outStream = null; + Writer writer = null; + try { + TocHeader tocHeader = new TocHeader(); + Map allFiles = new LinkedHashMap(); + int numberOfFilesTotal = 0; + int sizeOfTreFileNames = 0; + int sizeOfNameBlock = 0; + int numberOfTreFiles = 0; + int numberOfTOCTreFiles = 0; + String[] treFiles = new String[1000]; + for (int z = 1; z < args.length; z++) { + FileInputStream inputStream = new FileInputStream(args[z]); + String formquestionmark = readString(inputStream, 4); // decide here - TREE or TOC + if (formquestionmark.equalsIgnoreCase("eert")) { //TREE + sizeOfTreFileNames += (Paths.get(args[z]).getFileName().toString().length() + 1); + treFiles[numberOfTreFiles] = Paths.get(args[z]).getFileName().toString(); + numberOfTreFiles++; + String TemplateType = readString(inputStream, 4); // 1000 (4) + if(!(TemplateType.equalsIgnoreCase("5000") || TemplateType.equalsIgnoreCase("4000"))) { //we can read 4 or 5 TREs + System.out.println("Not a valid TREE File " + (Paths.get(args[z]).getFileName().toString())); + System.exit(124); + } + long number_of_files = readUint32(inputStream); // LENGTH (4) + long size_of_toc = readUint32(inputStream); // LENGTH (4) + long size_of_name_block = readUint32(inputStream); // LENGTH (4) + long size_of_name_block_uncon = readUint32(inputStream); // LENGTH (4) + long number_of_tre_files = readUint32(inputStream); // LENGTH (4) + long size_of_tre_block = readUint32(inputStream); // LENGTH (4) + long size_of_tre_blockt = readUint32(inputStream); // LENGTH (4) + System.out.println("TREE files + " + number_of_files); + TreeHeader header = new TreeHeader(formquestionmark, TemplateType, number_of_files, size_of_toc, + size_of_name_block, size_of_name_block_uncon, number_of_tre_files, size_of_tre_block, + size_of_tre_blockt); + numberOfFilesTotal += (int) header.numberOfFiles; + header.print(); + //System.exit(1); + sizeOfNameBlock += size_of_name_block; + } + else if(formquestionmark.equalsIgnoreCase(" cot")) { //TOC + String TemplateType = readString(inputStream, 4); //1000 (4) + if(!TemplateType.equalsIgnoreCase("1000")) { + System.out.println("Not a valid TOC File"); + System.exit(124); + } + skip(4, inputStream); //skip compressor & two unused (4) + long number_of_files = readUint32(inputStream); // LENGTH (4) + long size_of_toc = readUint32(inputStream); // LENGTH (4) + long size_of_name_block = readUint32(inputStream); // LENGTH (4) + long size_of_name_block_uncon = readUint32(inputStream); // LENGTH (4) + long number_of_tre_files = readUint32(inputStream); // LENGTH (4) + long size_of_tre_block = readUint32(inputStream); // LENGTH (4) + numberOfTOCTreFiles += number_of_tre_files; + System.out.println("TOC files + " + number_of_files); + + for(int i = 0; i < number_of_tre_files; i++) { //print out our tre file names + String treName = readUntil((byte)0,inputStream); + treFiles[numberOfTreFiles] = treName; + numberOfTreFiles++; + } + numberOfFilesTotal += (int) number_of_files; + sizeOfNameBlock += size_of_name_block_uncon; + sizeOfTreFileNames += size_of_tre_block; + + } + else { + System.out.println("Not a valid TRE or TOC file " + (Paths.get(args[z]).getFileName().toString())); + } + inputStream.close(); + } + tocHeader.setSizeOfTreFileNameBlock(sizeOfTreFileNames); // toc header - size of tre file name block + tocHeader.setNumberOfFiles(numberOfFilesTotal); // toc header - number of files + tocHeader.setSizeOfNameBlock(sizeOfNameBlock); // toc header - size of file name block + tocHeader.setSizeOfTOC(numberOfFilesTotal * 24); + tocHeader.setNumberOfTreFiles(numberOfTreFiles); // toc header - number of tre files + System.out.println("Total number of files " + numberOfFilesTotal); + outStream = new FileOutputStream(outFilename); + writer = new BufferedWriter(new OutputStreamWriter(outStream)); + TableOfContentsEntry[] tocEntries = new TableOfContentsEntry[numberOfFilesTotal]; + int tocEntriesListNum = 0; + int TOCNamesSoFar = 0; + for (int z = 0; z < args.length; z++) { + FileInputStream inputStream = new FileInputStream(args[z]); + String formquestionmark = readString(inputStream, 4); // " " C O T (4) TOC_ + if (formquestionmark.equalsIgnoreCase("eert")) { //TREE + String TemplateType = readString(inputStream, 4); // 1000 (4) + long number_of_files = readUint32(inputStream); // LENGTH (4) + long size_of_toc = readUint32(inputStream); // LENGTH (4) + long size_of_name_block = readUint32(inputStream); // LENGTH (4) + long size_of_name_block_uncon = readUint32(inputStream); // LENGTH (4) + long number_of_tre_files = readUint32(inputStream); // LENGTH (4) + long size_of_tre_block = readUint32(inputStream); // LENGTH (4) + long size_of_tre_blockt = readUint32(inputStream); // LENGTH (4) + TreeHeader header = new TreeHeader(formquestionmark, TemplateType, number_of_files, size_of_toc, + size_of_name_block, size_of_name_block_uncon, number_of_tre_files, size_of_tre_block, + size_of_tre_blockt); + header.print(); + if (TemplateType.equalsIgnoreCase("6000")) { // invalid header? + System.out.println("Skipping input file " + args[z] + " because it does not have a TOC"); //so v6000 TOC files do not have a header - we can NOT read them. alert but skip. + } else if (TemplateType.equalsIgnoreCase("5000") || TemplateType.equalsIgnoreCase("4000")) { // 4 and 5 use same processing, valid headers + //numberOfTreFiles++; + TreeTOCEntry[] treetoc = new TreeTOCEntry[(int) header.numberOfFiles]; + inputStream.getChannel().position(header.tocOffset); + if (header.tocCompressor == 2) { //compressed table of contents, unzip and process + byte[] tocData = new byte[(int) header.sizeOfTOC]; + inputStream.read(tocData, 0, (int) header.sizeOfTOC); + Inflater decompresser = new Inflater(); + decompresser.setInput(tocData, 0, (int) header.sizeOfTOC); + byte[] result = new byte[24 * (int) header.numberOfFiles]; + decompresser.inflate(result); + decompresser.end(); + int loc = 0; + for (int i = 0; i < header.numberOfFiles; i++) { + long crc = readUint32(result, loc); // LENGTH (4) + loc += 4; + long length = readUint32(result, loc); + loc += 4; + long offset = readUint32(result, loc); + loc += 4; + long compressor = readUint32(result, loc); + loc += 4; + long compressedLength = readUint32(result, loc); + loc += 4; + long fileNameOffset = readUint32(result, loc); + loc += 4; + treetoc[i] = new TreeTOCEntry(crc, length, offset, compressor, compressedLength, fileNameOffset); + //treetoc[i].print(); + } + } else { //uncompressed table of contents, just read + for (int i = 0; i < header.numberOfFiles; i++) { + long crc = readUint32(inputStream); // LENGTH (4) + long length = readUint32(inputStream); + long offset = readUint32(inputStream); + long compressor = readUint32(inputStream); + long compressedLength = readUint32(inputStream); + long fileNameOffset = readUint32(inputStream); + treetoc[i] = new TreeTOCEntry(crc, length, offset, compressor, compressedLength, fileNameOffset); + } + } + if (header.blockCompressor == 2) { //compressed name block, unzip and process + byte[] blockData = new byte[(int) header.sizeOfNameBlock]; + inputStream.read(blockData, 0, (int) header.sizeOfNameBlock); + Inflater decompresser = new Inflater(); + decompresser.setInput(blockData, 0, (int) header.sizeOfNameBlock); + byte[] result = new byte[(int) header.uncompSizeOfNameBlock]; + decompresser.inflate(result); + //System.out.println("ADLER" + decompresser.getAdler()); + decompresser.end(); + int loc = 0; + for (int i = 0; i < header.numberOfFiles; i++) { + String trename = readUntil((byte) 0, result, loc); + treetoc[i].setName(trename); + treetoc[i].print(); + System.out.println("LOCATION IS " + loc); + loc += trename.length() + 1; + } + } else { //uncompressed name block, just read + for (int i = 0; i < header.numberOfFiles; i++) { + String trename = readUntil((byte) 0, inputStream); + treetoc[i].setName(trename); + } + } + for (int j = 0; j < treetoc.length; j++) { //compile our separate tree's table of contents into one main table of contents for the TOC + tocEntries[tocEntriesListNum] = new TableOfContentsEntry(treetoc[j], numberOfTOCTreFiles); + tocEntriesListNum++; + } + numberOfTOCTreFiles++; //increase tre size number after we set what tre they are from + } else { + System.out.println("Tree File Error"); + System.exit(124); + } + } + else if(formquestionmark.equalsIgnoreCase(" cot")) { //TOC + String TemplateType = readString(inputStream,4); //1000 (4) + if(!TemplateType.equalsIgnoreCase("1000")) { + System.out.println("Not a valid TOC File"); //TOC only exists as 1000. bail if not. + System.exit(124); + } + skip(4, inputStream); //skip compressor & two unused (4) + long number_of_files = readUint32(inputStream); // LENGTH (4) + System.out.println("Reading a TOC..."); + System.out.println("Number of Files: " + number_of_files); + long size_of_toc = readUint32(inputStream); // LENGTH (4) + System.out.println("Size of TOC: " + size_of_toc); + long size_of_name_block = readUint32(inputStream); // LENGTH (4) + System.out.println("Size of File Names: " + size_of_name_block); + long size_of_name_block_uncon = readUint32(inputStream); // LENGTH (4) + System.out.println("Size of File Names (Uncompressed): " + size_of_name_block_uncon); + long number_of_tre_files = readUint32(inputStream); // LENGTH (4) + System.out.println("Number of TRE Files: " + number_of_tre_files); + long size_of_tre_block = readUint32(inputStream); // LENGTH (4) + System.out.println("Size of TRE Files List: " + size_of_tre_block); + System.out.println("Currently at position " + inputStream.getChannel().position()); + System.out.println("END Reading a TOC..."); + + TableOfContentsEntry[] tocEntriesFromATOC = new TableOfContentsEntry[(int) number_of_files]; + + for(int i = 0; i < number_of_tre_files; i++) { //print out our tre file names + readUntil((byte)0,inputStream); //skipping over these? print them later - before Z + } + + for(int i = 0; i keys = allFiles.keySet(); + for (int i = 0; i < tocEntries.length; i++) { + if (!tocEntries[i].skipThis) { //if we didn't say to skip this one, write it. skipped = duplicated / exists in older tre + outStream.write(tocEntries[i].toByteArray(), 0, 24); + } + } + long endPos = outStream.getChannel().position(); + System.out.println("Start and end of TOC: " + initialPos + ", " + endPos + ", x-y= " + (endPos - initialPos)); + initialPos = outStream.getChannel().position(); + System.out.println("writing filename " + tocEntries[0].fileName); + + //tocEntries[0].print(); + //tocEntries[1].print(); + //tocEntries[2].print(); + + for (int i = 0; i < tocEntries.length; i++) { + if (!tocEntries[i].skipThis) { + outStream.write(tocEntries[i].fileName.getBytes()); //writing file names and the null + outStream.write((byte) 0); + } + } + endPos = outStream.getChannel().position(); + System.out.println("Start and end of Name Block: " + initialPos + ", " + endPos + ", x-y= " + (endPos - initialPos)); + System.out.println("new number of files " + allFiles.size()); + + for (int z = 0; z < numberOfTreFiles; z++) { + System.out.println("TRE " + z + " is " + treFiles[z]); //for debug! + } + } catch (FileNotFoundException ex) { + System.out.println("Unable to open file"); + } catch (IOException ex) { + System.out.println("Error reading file"); + } catch (DataFormatException e) { // from inflater + e.printStackTrace(); + } + finally { + try { + writer.close(); + outStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } +} diff --git a/src/io/elour/TableOfContentsEntry.java b/src/io/elour/TableOfContentsEntry.java new file mode 100644 index 0000000..21c720d --- /dev/null +++ b/src/io/elour/TableOfContentsEntry.java @@ -0,0 +1,93 @@ +package io.elour; + +import java.io.IOException; +import java.io.Writer; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class TableOfContentsEntry implements Comparable{ //24 BYTES + int compressor; // 1 char + int unused; // 2 char + short treeFileIndex; // 3 4 + long crc; // 5 6 7 8 + long fileNameOffset; // 9 10 11 12 + long offset; // 13 14 15 16 + long length; // 17 18 19 20 + long compressedLength; // 21 22 23 24 + String fileName = ""; + boolean skipThis = false; + + TableOfContentsEntry(int compressor, int unused, short treeFileIndex, long crc, long fileNameOffset, long offset, long length, long compressedLength) { + this.compressor = compressor; + this.unused = unused; + this.treeFileIndex = treeFileIndex; + this.crc = crc; + this.fileNameOffset = fileNameOffset; + this.offset = offset; + this.length = length; + this.compressedLength = compressedLength; + } + + TableOfContentsEntry(TreeTOCEntry a, int treeFileIndex) { //create a TableOfContentsEntry from a TreeTOCEntry + this.compressor = (int) a.compressor; + this.unused = 0; + this.treeFileIndex = (short) treeFileIndex; + this.crc = a.crc; + this.fileNameOffset = a.name.length(); + this.offset = a.offset; + this.length = a.length; + this.compressedLength = a.compressedLength; + if(length != 0 && compressedLength == 0) { + compressedLength = length; + } + this.fileName = a.name; + this.skipThis = false; + } + + void print() { + System.out.println("compressor: " + compressor + ", unused: " + unused + ", tree file index: " + treeFileIndex + ", crc: " + crc + ", file name length: " + fileNameOffset + ", offset: " + offset + ", length: " + length + ", compressed length: " + compressedLength + ", file name " + fileName + ", skip? " + skipThis); + } + String tsvinfo() { + return "" + compressor + "\t" + treeFileIndex + "\t" + crc + "\t" + fileNameOffset + "\t" + offset + "\t" + length + "\t" + compressedLength + "\n"; + } + void writeData(Writer outputHere) { + try { + outputHere.write(""+compressor + unused + treeFileIndex + crc + fileNameOffset + offset + length + compressedLength); + } catch (IOException e) { + e.printStackTrace(); + } + } + byte[] toByteArray() { + ByteBuffer buffer = ByteBuffer.allocate(28); + buffer.order(ByteOrder.LITTLE_ENDIAN); //Important! silly SOE + buffer.put((byte) compressor); //1 + buffer.put((byte) unused); //2 + buffer.position(2); + buffer.putShort(treeFileIndex); //3 4 + buffer.putLong(crc); //5 6 7 8 + buffer.position(8); //longs are '8' bytes in java, ignore the last 4 bytes and rewind + buffer.putLong(fileNameOffset); //9 10 11 12 + buffer.position(12); + buffer.putLong(offset); //9 10 11 12 + buffer.position(16); + buffer.putLong(length); //13 14 15 16 + buffer.position(20); + buffer.putLong(compressedLength); //17 18 19 20 + //buffer = ByteBuffer.wrap(buffer.array(), 0, 24); //this doesn't do anything? so... + return buffer.array(); + } + + @Override + public int compareTo(TableOfContentsEntry arg0) { //implementing this, the sort is not automatic? be sure to call Arrays.sort()... + if(this.crc > arg0.crc) { + return 1; + } + else if(this.crc < arg0.crc) { + return -1; + } + else { + return 0; + } + } +} + diff --git a/src/io/elour/TocHeader.java b/src/io/elour/TocHeader.java new file mode 100644 index 0000000..5a8a859 --- /dev/null +++ b/src/io/elour/TocHeader.java @@ -0,0 +1,139 @@ +package io.elour; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class TocHeader { //32 bytes + String token; //1 2 3 4 + String version; //5 6 7 8 + short tocCompressor; // 9 + short fileNameCompressor; // 10 + short unusedone; // 11 + short unusedtwo; // 12 + long numberOfFiles; //13 14 15 16 + long sizeOfTOC; //17 18 19 20 + long sizeOfNameBlock; //21 22 23 24 + long uncompSizeOfNameBlock; //25 26 27 28 + long numberOfTreFiles; //29 30 31 32 + long sizeOfTreFileNameBlock; //33 34 35 36 + + TocHeader() { + this.token = ""; + this.version = "1000"; + this.tocCompressor = 0; + this.fileNameCompressor = 0; + this.unusedone = 0; + this.unusedtwo = 0; + } + + TocHeader(String a, String b, long c, long d, long e, long f, long g, long h) { + this.token = a; + this.version = b; + this.tocCompressor = 0; + this.fileNameCompressor = 0; + this.unusedone = 0; + this.unusedtwo = 0; + this.numberOfFiles = c; + this.sizeOfTOC = d; + this.sizeOfNameBlock = e; + this.uncompSizeOfNameBlock = f; + this.numberOfTreFiles = g; + this.sizeOfTreFileNameBlock = h; + } + + void setSizeOfTOC(long sizeOfTOC) { //are these really necessary? can't we just "identifier.variable = #" ? + this.sizeOfTOC = sizeOfTOC; + } + void setNumberOfFiles(long numberOfFiles) { + this.numberOfFiles = numberOfFiles; + } + void setSizeOfNameBlock(long numberOfFiles) { + this.sizeOfNameBlock = numberOfFiles; + this.uncompSizeOfNameBlock = numberOfFiles; + } + void setNumberOfTreFiles(long numberOfFiles) { + this.numberOfTreFiles = numberOfFiles; + } + void setSizeOfTreFileNameBlock(long numberOfFiles) { + this.sizeOfTreFileNameBlock = numberOfFiles; + } + + void print() { + System.out.println("TREE Info: Type "+token+", Version " + version + ", Number of Files: " + numberOfFiles + ", TOC Offset: " + 0 + + ", TOC Comprsesed? " + tocCompressor + ", Size of TOC: " + sizeOfTOC + ", Block Compressor? " + 0 + ", Size of Name Block: " + sizeOfNameBlock + + ", Uncompressed Size of Name Block: " + uncompSizeOfNameBlock + "\n"); + } + byte[] toByteArray() { + ByteBuffer buffer = ByteBuffer.allocate(32); + buffer.put((byte) tocCompressor); //1 + buffer.put((byte) fileNameCompressor); //2 + buffer.put((byte) unusedone); //3 + buffer.put((byte) unusedtwo); //4 + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.putLong(numberOfFiles); //5 6 7 8 + buffer.position(8); //longs are '8' bytes in java, ignore the last 4 bytes and rewind + buffer.putLong(sizeOfTOC); // 9 10 11 12 + buffer.position(12); + buffer.putLong(sizeOfNameBlock); // 13 14 15 16 + buffer.position(16); + buffer.putLong(uncompSizeOfNameBlock); //17 18 19 20 + buffer.position(20); + buffer.putLong(numberOfTreFiles); + buffer.position(24); + buffer.putLong(sizeOfTreFileNameBlock); + buffer.position(28); + buffer.limit(28); + return buffer.array(); + } + ByteBuffer getByteBuffer() { + ByteBuffer buffer = ByteBuffer.allocate(32); + buffer.put((byte) tocCompressor); //1 + buffer.put((byte) fileNameCompressor); //2 + buffer.put((byte) unusedone); //3 + buffer.put((byte) unusedtwo); //4 + buffer.putLong(numberOfFiles); //5 6 7 8 + buffer.position(8); //longs are '8' bytes in java, ignore the last 4 bytes and rewind + buffer.putLong(sizeOfTOC); // 9 10 11 12 + buffer.position(12); + buffer.putLong(sizeOfNameBlock); // 13 14 15 16 + buffer.position(16); + buffer.putLong(uncompSizeOfNameBlock); //17 18 19 20 + buffer.position(20); + buffer.putLong(numberOfTreFiles); + buffer.position(24); + buffer.putLong(sizeOfTreFileNameBlock); + buffer.position(28); + buffer.limit(28); + buffer = ByteBuffer.wrap(buffer.array(), 0, 28); + return buffer; + } + CharSequence toCharSequence() { + ByteBuffer buffer = ByteBuffer.allocate(32); + buffer.put((byte) tocCompressor); //1 + buffer.put((byte) fileNameCompressor); //2 + buffer.put((byte) unusedone); //3 + buffer.put((byte) unusedtwo); //4 + buffer.putLong(numberOfFiles); //5 6 7 8 + buffer.position(8); //longs are '8' bytes in java, ignore the last 4 bytes and rewind + buffer.putLong(sizeOfTOC); // 9 10 11 12 + buffer.position(12); + buffer.putLong(sizeOfNameBlock); // 13 14 15 16 + buffer.position(16); + buffer.putLong(uncompSizeOfNameBlock); //17 18 19 20 + buffer.position(20); + buffer.putLong(numberOfTreFiles); + buffer.position(24); + buffer.putLong(sizeOfTreFileNameBlock); + buffer.position(28); + buffer.limit(28); + CharSequence x = null; + try { + x = new String(buffer.array(), "US-ASCII"); + } catch (UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return x; + } +} diff --git a/src/io/elour/TreeHeader.java b/src/io/elour/TreeHeader.java new file mode 100644 index 0000000..cc20ecf --- /dev/null +++ b/src/io/elour/TreeHeader.java @@ -0,0 +1,59 @@ +package io.elour; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class TreeHeader { // 36 bytes + String token; //1 2 3 4 + String version; //5 6 7 8 + long numberOfFiles; //9 10 11 12 + long tocOffset; //13 14 15 16 + long tocCompressor; //17 18 19 20 + long sizeOfTOC; //21 22 23 24 + long blockCompressor; //25 26 27 28 + long sizeOfNameBlock; //29 30 31 32 + long uncompSizeOfNameBlock; //33 34 35 36 + + //Token, Version, Num Files, TOC Offset, TOC Comp, TOC Size, Block Comp, Block Size, Uncomp Block Size + TreeHeader(String a, String b, long c, long d, long e, long f, long g, long h, long i) { + this.token = a; + this.version = b; + this.numberOfFiles = c; + this.tocOffset = d; + this.tocCompressor = e; + this.sizeOfTOC = f; + this.blockCompressor = g; + this.sizeOfNameBlock = h; + this.uncompSizeOfNameBlock = i; + } + + void print() { + System.out.println("TREE Info: Type "+token+", Version " + version + ", Number of Files: " + numberOfFiles + ", TOC Offset: " + tocOffset + + ", TOC Comprsesed? " + tocCompressor + ", Size of TOC: " + sizeOfTOC + ", Block Compressor? " + blockCompressor + ", Size of Name Block: " + sizeOfNameBlock + + ", Uncompressed Size of Name Block: " + uncompSizeOfNameBlock + "\n"); + } + byte[] toByteArray() { + ByteBuffer buffer = ByteBuffer.allocate(44); + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.put(this.token.getBytes()); // 1 2 3 4 + buffer.position(4); + buffer.put(this.version.getBytes()); // 5 6 7 8 + buffer.position(8); + buffer.putLong(numberOfFiles); //9 10 11 12 + buffer.position(12); + buffer.putLong(tocOffset); // 13 14 15 16 + buffer.position(16); + buffer.putLong(tocCompressor); // 17 18 19 20 + buffer.position(20); + buffer.putLong(sizeOfTOC); //21 22 23 24 + buffer.position(24); + buffer.putLong(blockCompressor);// 25 26 27 28 + buffer.position(28); + buffer.putLong(sizeOfNameBlock); //29 30 31 32 + buffer.position(32); + buffer.putLong(uncompSizeOfNameBlock); //33 34 35 36 + buffer.position(36); + buffer.limit(36); + return buffer.array(); + } +} \ No newline at end of file diff --git a/src/io/elour/TreeTOCEntry.java b/src/io/elour/TreeTOCEntry.java new file mode 100644 index 0000000..5f03c41 --- /dev/null +++ b/src/io/elour/TreeTOCEntry.java @@ -0,0 +1,57 @@ +package io.elour; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class TreeTOCEntry implements Comparable { //24 BYTES + long crc; // 1 2 3 4 + long length; // 5 6 7 8 + long offset; // 9 10 11 12 + long compressor; // 13 14 15 16 + long compressedLength; // 17 18 19 20 + long fileNameOffset; // 21 22 23 24 + long originalFileNameOffset; + String name; + + TreeTOCEntry(long crc, long length, long offset, long compressor, long compressedLength, long fileNameOffset) { + this.compressor = compressor; + this.crc = crc; + this.fileNameOffset = fileNameOffset; + this.offset = offset; + this.length = length; + this.compressedLength = compressedLength; + this.name = ""; + } + + void print() { + System.out.println("compressor: " + compressor + ", crc: " + crc + ", file name length: " + fileNameOffset + ", file name offset: " + originalFileNameOffset + ", offset: " + offset + ", length: " + length + ", compressed length: " + compressedLength + ", File name: " + name); + } + + void setName(String name) { + this.name = name; + originalFileNameOffset = fileNameOffset; + this.fileNameOffset = name.length(); + } + byte[] toByteArray() { + ByteBuffer buffer = ByteBuffer.allocate(30); + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.putLong(this.crc); // 1 2 3 4 + buffer.position(4); + buffer.putLong(this.length); // 5 6 7 8 + buffer.position(8); + buffer.putLong(this.offset); //9 10 11 12 + buffer.position(12); + buffer.putLong(this.compressor); // 13 14 15 16 + buffer.position(16); + buffer.putLong(this.compressedLength); // 17 18 19 20 + buffer.position(20); + buffer.putLong(this.originalFileNameOffset); //21 22 23 24 + buffer.limit(24); + return buffer.array(); + } + + @Override + public int compareTo(TreeTOCEntry arg0) { + return Long.compare(this.crc, arg0.crc); + } +} \ No newline at end of file diff --git a/src/io/elour/Utils.java b/src/io/elour/Utils.java new file mode 100644 index 0000000..93033d1 --- /dev/null +++ b/src/io/elour/Utils.java @@ -0,0 +1,517 @@ +package io.elour; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +@SuppressWarnings("ResultOfMethodCallIgnored") +public class Utils { + /** + * Skips ahead in an input stream + * @param skipNum Number of bytes to skip + * @param toSkip FileInputStream to perform reads on + */ + static void skip(int skipNum, InputStream toSkip) + { + try + { + long skipped = toSkip.skip(skipNum); + if(skipped != skipNum) + { + System.err.println("Should have skipped " + skipNum + " but we skipped " + skipped); + skip((int) (skipNum - skipped), toSkip); + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + static void skipReal(int skipNum, InputStream toSkip) + { + try + { + byte[] b = new byte[skipNum]; + toSkip.read(b, 0, skipNum); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + /** + * Returns the length of a FORM or CHUNK (4 bytes) + * @param toSkip FileInputStream to perform reads on + */ + static int getLength(InputStream toSkip) { + try { + return (toSkip.read() << 24 | (toSkip.read() << 16) | (toSkip.read() << 8) | (toSkip.read())); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return -1; + } + + /** + * Reads a short (2 bytes) + * @param toSkip FileInputStream to perform reads on + */ + static int readShort(InputStream toSkip) { + try { + return ((toSkip.read()) | (toSkip.read() << 8)); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return -1; + } + + /** + * Reads a float (4 bytes) + * @param toSkip FileInputStream to perform reads on + */ + static float floatFetcher(InputStream toSkip) { + try { + byte[] b = new byte[4]; + toSkip.read(b, 0, 4); + return ByteBuffer.wrap(Objects.requireNonNull(b)).order(ByteOrder.LITTLE_ENDIAN).getFloat(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return -1f; + } + + /** + * Reads an unsigned integer (4 bytes) + * @param toSkip FileInputStream to perform reads on + */ + static long readBigEndian4(InputStream toSkip) { + try { + byte[] b = new byte[4]; + toSkip.read(b, 0, 4); + //System.err.println("Big Endian byte array: " + b[0] +" "+ b[1] +" "+ b[2] +" "+ b[3]); + ByteBuffer bb = ByteBuffer.wrap(b); + //bb.order(ByteOrder.LITTLE_ENDIAN); + return bb.getInt(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return -1; + } + + /** + * Reads an unsigned integer (4 bytes) + * @param toSkip FileInputStream to perform reads on + */ + static long readUint32(InputStream toSkip) { + try { + byte[] b = new byte[4]; + toSkip.read(b, 0, 4); + return ((long) (b[3] & 0xFF) << 24) | (b[2] & 0xFF) << 16 | (b[1] & 0xFF) << 8 | b[0] & 0xFF; + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return -1; + } + + /** + * Reads an unsigned integer from a byte array + * @param input byte array containing integer + * @param loc length of uint to read + * @return the int + */ + static long readUint32(byte[] input, int loc) { + return ((long) (input[loc + 3] & 0xFF) << 24) | (long) ((input[loc + 2] & 0xFF) << 16) + | (long) ((input[loc + 1] & 0xFF) << 8) | (long) (input[loc] & 0xFF); + } + static long readUint32(byte[] input) + { + return readUint32(input, 0); + } + + /** + * Reads a reverse unsigned integer (4 bytes) + * @param toSkip FileInputStream to perform reads on + */ + static long readRUint(InputStream toSkip) { + try { + return ((long) (toSkip.read()) & 0xff | (long) (toSkip.read() << 8) & 0xff00 + | (long) (toSkip.read() << 16) & 0xff0000 | (long) (toSkip.read() << 24) & 0xff000000); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return -1; + } + + /** + * Reads data until a specified delimiter is seen + * @param delim Delimiter to read until + * @param toSkip FileInputStream to perform reads on + */ + static String readUntil(byte delim, InputStream toSkip) { + try { + byte thisRead = (byte) toSkip.read(); + StringBuilder returnme = new StringBuilder(); + while (thisRead != delim) { + returnme.append((char) thisRead); + thisRead = (byte) toSkip.read(); + } + return returnme.toString(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return "err"; + } + + /** + * Reads data until a specified delimiter is seen + * @param delim Delimiter to read until + * @param input An input byte array to read from + * @param loc Start point to read from in input + */ + static String readUntil(byte delim, byte[] input, int loc) { + StringBuilder returnme = new StringBuilder(); + while (input[loc] != delim) { + returnme.append((char) input[loc]); + loc++; + } + return returnme.toString(); + } + + /** + * Reads a string from IFF files + * @param toSkip FileInputStream to perform reads on + * @param length Length of String to read + */ + static String readString(InputStream toSkip, int length) { + String baseObject = ""; + try { + byte[] baseObjectAr = new byte[length]; // create a byte array of + // length + toSkip.read(baseObjectAr, 0, length); + baseObject = new String(baseObjectAr, StandardCharsets.UTF_8); // turn the byte + // array into a + // string + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } // read the bytes + + return baseObject; + } + + /** + * Reads a string from IFF files + * @param toSkip FileInputStream to perform reads on + */ + static String readString(InputStream toSkip) { + return readUntil((byte)0, toSkip); + } + + /** + * Reads a unicode string from IFF files + * @param toSkip FileInputStream to perform reads on + * @param length Length of String to read + */ + static String readWideString(InputStream toSkip, int length) { + String baseObject = ""; + try { + byte thisRead; + StringBuilder returnme = new StringBuilder(); + for(int i = 0; i < length; i++) + { + thisRead = (byte) toSkip.read(); + returnme.append((char) thisRead); + toSkip.read(); //throwaway + } + return returnme.toString(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } // read the bytes + + return baseObject; + } + + /** + * Checks for a corresponding string set (FORM, 0001 etc) + * @param toCheckFor The answer we're looking for + * @param inputStream FileInputStream to perform reads on + * @param fatalOnFalse If true, exit program on failure to match + */ + public static boolean checkFor(String toCheckFor, InputStream inputStream, boolean fatalOnFalse) + { + String compare = readString(inputStream, 4); + return compare.equalsIgnoreCase(toCheckFor); + } + /** + * Checks for a corresponding string set (FORM, 0001 etc) + * Exits program if match fails + * @param toCheckFor The answer we're looking for + * @param inputStream FileInputStream to perform reads on + */ + public static void checkFor(String toCheckFor, InputStream inputStream) + { + String compare = readString(inputStream, 4); + if(!compare.equalsIgnoreCase(toCheckFor)) + { + System.err.println("CheckFor failed, " + toCheckFor + " != " + compare); + System.exit(123); + } + } + + /** + * Reads data from an input stream and prints out the data in a hex-dump format + * @param is The input stream + * @throws IOException err0r + */ + public static void hexDump(InputStream is) throws IOException { + int i = 0; + + while (is.available() > 0) { + StringBuilder sb1 = new StringBuilder(); + StringBuilder sb2 = new StringBuilder(" "); + System.out.printf("%04X ", i * 16); + for (int j = 0; j < 16; j++) { + if (is.available() > 0) { + int value = is.read(); + sb1.append(String.format("%02X ", value)); + if (!Character.isISOControl(value)) { + sb2.append((char)value); + } + else { + sb2.append("."); + } + } + else { + for (;j < 16;j++) { + sb1.append(" "); + } + } + } + System.out.print(sb1); + System.out.println(sb2); + i++; + } + is.close(); + } + + /** + * Looks up the specified template from a FORM name + * @param idInput the FORM name input + * @return a client or server tdf filename + */ + static String templateLookup(String idInput) { + if (idInput.equalsIgnoreCase("SBMK")) { + return "battlefield_marker_object_template"; + } + + if (idInput.equalsIgnoreCase("SBOT")) { + return "building_object_template"; + } + + if (idInput.equalsIgnoreCase("CCLT")) { + return "cell_object_template"; + } + + if (idInput.equalsIgnoreCase("SCNC")) { + return "construction_contract_object_template"; + } + + if (idInput.equalsIgnoreCase("SCOT")) { + return "creature_object_template"; + } + + if (idInput.equalsIgnoreCase("SDSC")) { + return "draft_schematic_object_template"; + } + + if (idInput.equalsIgnoreCase("SFOT")) { + return "factory_object_template"; + } + + if (idInput.equalsIgnoreCase("SGRP")) { + return "group_object_template"; + } + + if (idInput.equalsIgnoreCase("SGLD")) { + return "guild_object_template"; + } + + if (idInput.equalsIgnoreCase("SIOT")) { + return "installation_object_template"; + } + + if (idInput.equalsIgnoreCase("SITN")) { + return "intangible_object_template"; + } + + if (idInput.equalsIgnoreCase("SJED")) { + return "jedi_manager_object_template"; + } + + if (idInput.equalsIgnoreCase("SMSC")) { + return "manufacture_schematic_object_template"; + } + + if (idInput.equalsIgnoreCase("SMSD")) { + return "mission_data_object_template"; + } + + if (idInput.equalsIgnoreCase("SMLE")) { + return "mission_list_entry_object_template"; + } + + if (idInput.equalsIgnoreCase("SMSO")) { + return "mission_object_template"; + } + + if (idInput.equalsIgnoreCase("SHOT")) { + return "object_template"; + } + + if (idInput.equalsIgnoreCase("SPLY")) { + return "player_object_template"; + } + + if (idInput.equalsIgnoreCase("SPQO")) { + return "player_quest_object_template"; + } + + if (idInput.equalsIgnoreCase("RCCT")) { + return "resource_container_object_template"; + } + + if (idInput.equalsIgnoreCase("SSHP")) { + return "ship_object_template"; + } + + if (idInput.equalsIgnoreCase("STAT")) { + return "static_object_template"; + } + + if (idInput.equalsIgnoreCase("STOT")) { + return "tangible_object_template"; + } + + if (idInput.equalsIgnoreCase("STOK")) { + return "token_object_template"; + } + + if (idInput.equalsIgnoreCase("SUNI")) { + return "universe_object_template"; + } + + if (idInput.equalsIgnoreCase("SVOT")) { + return "vehicle_object_template"; + } + + if (idInput.equalsIgnoreCase("SWAY")) { + return "waypoint_object_template"; + } + + if (idInput.equalsIgnoreCase("SWOT")) { + return "weapon_object_template"; + } + + return "UNKNOWN"; + } + + /** + * Returns a gender string based on input + * @param input A gender type as integer + * @return String of the gender input + */ + static String genderLookup(int input) { + if (input == 0) + return "GE_male"; + if (input == 1) + return "GE_female"; + if (input == 2) + return "GE_other"; + return "UNKNOWN"; + } + + /** + * Returns a species string based on input + * @param input A species type as integer + * @return String of the species input + */ + static String speciesLookup(int input) { + if (input == 12) + return "SP_bith"; + return "UNKNOWN"; + } + + /** + * Returns data from the text of a template definition + * @param fileContent Text of the template definition file + * @param toLook Type of data to look for + * @param data unused? 0 + * @return Information of tdf from toLook + */ + static String templateLookupDataFrom(String fileContent, String toLook, int data) { + int startLoc = fileContent.lastIndexOf("version "); + + int whereToStart = fileContent.toLowerCase().indexOf(toLook.toLowerCase(), startLoc); + whereToStart += toLook.length(); + whereToStart = fileContent.indexOf("{", whereToStart) + 1; + List list = new ArrayList(); + String str = ""; + while ((str = fileContent.substring(whereToStart, fileContent.indexOf("\n", whereToStart))) != null) { + if (str.trim().equalsIgnoreCase("")) { + whereToStart = fileContent.indexOf("\n", whereToStart) + 1; + continue; + } + if (str.trim().charAt(0) == '/') { + whereToStart = fileContent.indexOf("\n", whereToStart) + 1; + continue; + } + if (str.trim().equalsIgnoreCase("}")) { + break; + } + list.add(str.trim()); + whereToStart = fileContent.indexOf("\n", whereToStart) + 1; + } + return ""; + } + + /** + * Returns data from an in-repo file + * @param rsc String path of the file to read from + * @return Text data of the file input + */ + static String getResource(String rsc) { + StringBuilder val = new StringBuilder(); + + try { + Class cls = Class.forName("Utils"); + + // returns the ClassLoader object associated with this Class + ClassLoader cLoader = cls.getClassLoader(); + + // input stream + InputStream i = cLoader.getResourceAsStream(rsc); + BufferedReader r = new BufferedReader(new InputStreamReader(i)); + + // reads each line + String l; + while((l = r.readLine()) != null) { + val.append(l); + } + i.close(); + } catch(Exception e) { + System.out.println(e); + } + return val.toString(); + } +}