Commits

Mathias Panzenböck committed 2d18db2

print more stats (maybe rename --coverage to --stats?)

Comments (0)

Files changed (8)

 
 For vpkfs [FUSE][1] is needed.
 
+File Format
+-----------
+This is a short description of the Valve Packet file format (VPK).
+
+### `*_dir.vpk`
+To my knowlege there are currently (as of Portal 2) two versions of the VPK
+file format. The only difference is that one has a file magic, version
+information and an explicitely given index size, and the other (older) don't.
+
+All values are stored in **little endian**. Offset and Size are given in
+bytes.
+
+#### Types
+
+	Byte ..... a single byte
+	U32 ...... unsigned 32-bit integer
+	ASCIIZ ... zero terminated ASCII string
+
+#### File header (optional)
+If the file doesn't start wit the file magic `0x55AA1234` then the index
+table directly starts at the beginning of the file.
+
+	Offset  Size  Type  Description
+	     0     4  U32   file magic: 0x55AA1234
+	     4     4  U32   version: 1
+	     8     4  U32   index size
+
+#### Index
+Files are grouped by their type (file name extension).
+	
+	Offset  Size  Type    Description
+	[
+	     0     N  ASCIIZ  File type
+		 
+	]*
+	           1  Byte    List-Terminator: 0x00
+
+
+
+### `*_###.vpk`
+These are simple archives where the contained files are simply merged together
+in one big file. You can think of them being created like this:
+
+	cat file1 file2 file3 > pak01_001.vpk
+
 Notes
 -----
 This project is written by Mathias Panzenböck and released under the LGPL v2.1.

unvpk/CMakeLists.txt

 
 add_executable(unvpk
 	src/main.cpp
+	src/archive_stat.cpp
 	src/coverage.cpp
 	src/console_table.cpp
 	src/magic.cpp

unvpk/include/vpk/archive_stat.h

+/**
+ * unvpk - list, check and extract vpk archives
+ * Copyright (C) 2013  Mathias Panzenböck <grosser.meister.morti@gmx.net>
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+#ifndef VPK_ARCHIVE_STATS_H
+#define VPK_ARCHIVE_STATS_H
+
+#include <vpk/coverage.h>
+#include <vpk/file.h>
+
+namespace Vpk {
+	class ArchiveStat {
+	public:
+		ArchiveStat() :
+			m_files(0),
+			m_minPreload(-1),
+			m_maxPreload(0),
+			m_sumPreload(0),
+			m_minSize(-1),
+			m_maxSize(0),
+			m_sumSize(0) {}
+
+		void add(const File &file);
+
+		Coverage &coverage() { return m_coverage; }
+		const Coverage &coverage() const { return m_coverage; }
+		size_t files() const { return m_files; }
+		size_t minPreload() const { return m_minPreload; }
+		size_t maxPreload() const { return m_maxPreload; }
+		size_t sumPreload() const { return m_sumPreload; }
+		size_t minSize() const { return m_minSize; }
+		size_t maxSize() const { return m_maxSize; }
+		size_t sumSize() const { return m_sumSize; }
+
+	private:
+		Coverage m_coverage;
+		size_t   m_files;
+		size_t   m_minPreload;
+		size_t   m_maxPreload;
+		size_t   m_sumPreload;
+		size_t   m_minSize;
+		size_t   m_maxSize;
+		size_t   m_sumSize;
+	};
+}
+
+#endif

unvpk/include/vpk/console_table.h

 /**
  * unvpk - list, check and extract vpk archives
- * Copyright (C) 2011  Mathias Panzenböck <grosser.meister.morti@gmx.net>
+ * Copyright (C) 2011-2013  Mathias Panzenböck <grosser.meister.morti@gmx.net>
  * 
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
 			CENTER,
 			RIGHT
 		};
-	
+
 		typedef std::vector<Alignment>   Alignments;
 		typedef std::vector<std::string> Row;
 		typedef std::vector<Row>         Table;
-	
-		ConsoleTable() : m_columns(0) {}
-	
+
+		ConsoleTable() : m_columns(0), m_delim("  ") {}
+
 		void addColumn(Alignment col);
 		void columns(Alignment col1);
 		void columns(Alignment col1, Alignment col2);
 		void columns(Alignment col1, Alignment col2, Alignment col3, Alignment col4);
 		void columns(Alignment col1, Alignment col2, Alignment col3, Alignment col4, Alignment col5);
 		void columns(Alignment col1, Alignment col2, Alignment col3, Alignment col4, Alignment col5, Alignment col6);
+		void columns(Alignment col1, Alignment col2, Alignment col3, Alignment col4, Alignment col5, Alignment col6, Alignment col7);
+		void columns(Alignment col1, Alignment col2, Alignment col3, Alignment col4, Alignment col5, Alignment col6, Alignment col7, Alignment col8);
 		void columns(const Alignments &alignments);
 
 		template<typename Type1>
 				m_columns = row.size();
 			}
 		}
-		
+
 		template<
 			typename Type1,
 			typename Type2>
 				m_columns = row.size();
 			}
 		}
-		
+
 		template<
 			typename Type1,
 			typename Type2,
 				m_columns = row.size();
 			}
 		}
-		
+
 		template<
 			typename Type1,
 			typename Type2,
 				m_columns = row.size();
 			}
 		}
-		
+
 		template<
 			typename Type1,
 			typename Type2,
 				m_columns = row.size();
 			}
 		}
-	
+
 		template<
 			typename Type1,
 			typename Type2,
 				m_columns = row.size();
 			}
 		}
+
+		template<
+			typename Type1,
+			typename Type2,
+			typename Type3,
+			typename Type4,
+			typename Type5,
+			typename Type6,
+			typename Type7>
+		void row(
+			const Type1 &col1,
+			const Type2 &col2,
+			const Type3 &col3,
+			const Type4 &col4,
+			const Type5 &col5,
+			const Type6 &col6,
+			const Type7 &col7) {
+			m_body.push_back(Row());
 	
+			Row &row = m_body.back();
+			row.push_back(boost::lexical_cast<std::string>(col1));
+			row.push_back(boost::lexical_cast<std::string>(col2));
+			row.push_back(boost::lexical_cast<std::string>(col3));
+			row.push_back(boost::lexical_cast<std::string>(col4));
+			row.push_back(boost::lexical_cast<std::string>(col5));
+			row.push_back(boost::lexical_cast<std::string>(col6));
+			row.push_back(boost::lexical_cast<std::string>(col7));
+	
+			if (m_columns < row.size()) {
+				m_columns = row.size();
+			}
+		}
+
+		template<
+			typename Type1,
+			typename Type2,
+			typename Type3,
+			typename Type4,
+			typename Type5,
+			typename Type6,
+			typename Type7,
+			typename Type8>
+		void row(
+			const Type1 &col1,
+			const Type2 &col2,
+			const Type3 &col3,
+			const Type4 &col4,
+			const Type5 &col5,
+			const Type6 &col6,
+			const Type7 &col7,
+			const Type8 &col8) {
+			m_body.push_back(Row());
+	
+			Row &row = m_body.back();
+			row.push_back(boost::lexical_cast<std::string>(col1));
+			row.push_back(boost::lexical_cast<std::string>(col2));
+			row.push_back(boost::lexical_cast<std::string>(col3));
+			row.push_back(boost::lexical_cast<std::string>(col4));
+			row.push_back(boost::lexical_cast<std::string>(col5));
+			row.push_back(boost::lexical_cast<std::string>(col6));
+			row.push_back(boost::lexical_cast<std::string>(col7));
+			row.push_back(boost::lexical_cast<std::string>(col8));
+	
+			if (m_columns < row.size()) {
+				m_columns = row.size();
+			}
+		}
+
 		void row(const Row &row);
 		void fillAlignments(Alignment alignment = LEFT);
 		void fillAlignments(size_t columns, Alignment alignment = LEFT);
 				bool truncate = true,
 				char fill = ' ');
 	
-		size_t            columns()    const { return m_columns; }
-		const Alignments &alignments() const { return m_alignments; }
-		const Table      &body()       const { return m_body; }
+		size_t             columns()    const { return m_columns; }
+		const Alignments  &alignments() const { return m_alignments; }
+		const Table       &body()       const { return m_body; }
+		const std::string &delimeter()  const { return m_delim; }
+
+		void setDelimeter(const std::string &delimeter) { m_delim = delimeter; }
 	
 		void clear();
 	
 	private:
-		size_t     m_columns;
-		Alignments m_alignments;
-		Table      m_body;
+		size_t      m_columns;
+		Alignments  m_alignments;
+		Table       m_body;
+		std::string m_delim;
 	};
 }
 

unvpk/include/vpk/coverage.h

 /**
  * unvpk - list, check and extract vpk archives
- * Copyright (C) 2011  Mathias Panzenböck <grosser.meister.morti@gmx.net>
+ * Copyright (C) 2011-2013  Mathias Panzenböck <grosser.meister.morti@gmx.net>
  * 
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public

unvpk/src/archive_stat.cpp

+/**
+ * unvpk - list, check and extract vpk archives
+ * Copyright (C) 2013  Mathias Panzenböck <grosser.meister.morti@gmx.net>
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include <vpk/archive_stat.h>
+
+void Vpk::ArchiveStat::add(const File &file) {
+	size_t preload = file.preload.size();
+	size_t size = preload + file.size;
+	m_sumPreload += preload;
+	m_sumSize    += size;
+	if (m_files == 0) {
+		m_minPreload = m_maxPreload = preload;
+		m_minSize    = m_maxSize    = size;
+	}
+	else {
+		if (preload > m_maxPreload) m_maxPreload = preload;
+		if (preload < m_minPreload) m_minPreload = preload;
+		
+		if (size > m_maxSize) m_maxSize = size;
+		if (size < m_minSize) m_minSize = size;
+	}
+	++ m_files;
+	m_coverage.add(file.offset, file.size);
+}

unvpk/src/console_table.cpp

 /**
  * unvpk - list, check and extract vpk archives
- * Copyright (C) 2011  Mathias Panzenböck <grosser.meister.morti@gmx.net>
+ * Copyright (C) 2011-2013  Mathias Panzenböck <grosser.meister.morti@gmx.net>
  * 
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
 	addColumn(col6);
 }
 
+void Vpk::ConsoleTable::columns(
+		Alignment col1,
+		Alignment col2,
+		Alignment col3,
+		Alignment col4,
+		Alignment col5,
+		Alignment col6,
+		Alignment col7) {
+	m_alignments.clear();
+	addColumn(col1);
+	addColumn(col2);
+	addColumn(col3);
+	addColumn(col4);
+	addColumn(col5);
+	addColumn(col6);
+	addColumn(col7);
+}
+
+void Vpk::ConsoleTable::columns(
+		Alignment col1,
+		Alignment col2,
+		Alignment col3,
+		Alignment col4,
+		Alignment col5,
+		Alignment col6,
+		Alignment col7,
+		Alignment col8) {
+	m_alignments.clear();
+	addColumn(col1);
+	addColumn(col2);
+	addColumn(col3);
+	addColumn(col4);
+	addColumn(col5);
+	addColumn(col6);
+	addColumn(col7);
+	addColumn(col8);
+}
+
 void Vpk::ConsoleTable::columns(const Alignments &alignments) {
 	m_alignments = alignments;
 	if (m_alignments.size() > m_columns) {
 	for (Table::const_iterator i = m_body.begin(); i != m_body.end(); ++ i) {
 		const Row &row = *i;
 		for (size_t j = 0, n = row.size(); j < n; ++ j) {
+			const std::string &cell = row[j];
+			Alignment alignment = alignments[j];
+
 			if (j + 1 < n) {
-				align(os, row[j], colsizes[j], alignments[j]);
-				os.put(' ');
+				align(os, cell, colsizes[j], alignment);
+				os << m_delim;
 			}
 			else {
-				align(os, row[j], row[j].size(), alignments[j]);
+				align(os, cell, alignment == LEFT ? cell.size() : colsizes[j], alignment);
 			}
 		}
 		os.put('\n');

unvpk/src/main.cpp

 /**
  * unvpk - list, check and extract vpk archives
- * Copyright (C) 2011  Mathias Panzenböck <grosser.meister.morti@gmx.net>
+ * Copyright (C) 2011-2013  Mathias Panzenböck <grosser.meister.morti@gmx.net>
  * 
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
 #include <vpk/util.h>
 #include <vpk/console_handler.h>
 #include <vpk/console_table.h>
+#include <vpk/archive_stat.h>
 #include <vpk/coverage.h>
 #include <vpk/magic.h>
 #include <vpk/list_entry.h>
 	std::cout << " total size), " << dirs << " " << (dirs == 1 ? "directory" : "directories") << "\n";
 }
 
-typedef std::map<int,Coverage> Coverages;
+typedef std::map<int,ArchiveStat> Stats;
 
-static void coverage(const Dir &dir, Coverages &covs) {
+static void archive_stat(const Dir &dir, Stats &stats) {
 	for (Dir::const_iterator i = dir.begin(); i != dir.end(); ++ i) {
 		Node *node = i->second.get();
 		if (node->type() == Node::DIR) {
-			coverage(*(Dir*) node, covs);
+			archive_stat(*(Dir*) node, stats);
 		}
 		else {
 			File *file = (File*) node;
-			if (file->size) {
-				covs[file->index].add(file->offset, file->size);
-			}
+			stats[file->index].add(*file);
 		}
 	}
 }
 		const Package &package,
 		bool dump,
 		const fs::path &destdir,
-		bool humanreadable) {
-	Coverages covs;
-	
-	covs[0x7fff].add(0, package.dataoff());
+		bool humanreadable,
+		bool printall) {
+	Stats stats;
+
+	stats[0x7fff].coverage().add(0, package.dataoff());
 
 	std::string prefix = tolower(package.name());
 	prefix += '_';
 			if (digits.size() >= 3) {
 				try {
 					uint16_t index = boost::lexical_cast<uint16_t>(digits);
-					covs[index];
+					stats[index];
 				}
 				catch (const boost::bad_lexical_cast&) {}
 			}
 		}
 	}
 
-	coverage(package, covs);
+	archive_stat(package, stats);
 
 	if (dump) {
 		create_path(destdir);
 	}
 
 	ConsoleTable statsTbl;
-	statsTbl.columns(ConsoleTable::LEFT, ConsoleTable::RIGHT, ConsoleTable::RIGHT, ConsoleTable::RIGHT, ConsoleTable::RIGHT);
-	statsTbl.row("File", "Size", "Covered", "%", "Missing", "Missing Areas");
+	statsTbl.columns(ConsoleTable::LEFT, ConsoleTable::RIGHT, ConsoleTable::RIGHT,
+	                 ConsoleTable::RIGHT, ConsoleTable::RIGHT, ConsoleTable::RIGHT,
+	                 ConsoleTable::LEFT);
+	statsTbl.row("File", "Files", "Size", "Covered", "%", "Missing", "Missing Areas");
 
+	size_t files = 0;
+	size_t minPreload = -1;
+	size_t maxPreload = 0;
+	size_t sumPreload = 0;
+	size_t minSize = -1;
+	size_t maxSize = 0;
+	size_t sumSize = 0;
 	size_t uncovered = 0;
 	size_t total = 0;
 	const size_t magicSize = Magic::maxSize();
 	std::vector<char> magic(magicSize, 0);
-	for (Coverages::const_iterator i = covs.begin(); i != covs.end(); ++ i) {
+	for (Stats::const_iterator i = stats.begin(); i != stats.end(); ++ i) {
 		fs::path path = package.archivePath(i->first);
 		std::string archive = path.filename().string();
 		size_t size = fs::file_size(path);
 		total += size;
 		std::string sizeStr = sizeToString(size, humanreadable);
 
-		const Coverage &covered = i->second;
+		const ArchiveStat &archStat = i->second;
+		const Coverage &covered = archStat.coverage();
+
+		files += archStat.files();
+		if (archStat.minPreload() < minPreload) minPreload = archStat.minPreload();
+		if (archStat.maxPreload() > maxPreload) maxPreload = archStat.maxPreload();
+
+		if (archStat.minSize() < minSize) minSize = archStat.minSize();
+		if (archStat.maxSize() > maxSize) maxSize = archStat.maxSize();
+		
+		sumPreload += archStat.sumPreload();
+		sumSize += archStat.sumSize();
 		Coverage missing = covered.invert(size);
-		
 		size_t missingSize = missing.coverage();
-		
-		if (missingSize == 0)
+
+		if (!printall && missingSize == 0)
 			continue;
 
 		uncovered += missingSize;
 		size_t coveredSize = covered.coverage();
 		std::string coveredSizeStr = sizeToString(coveredSize, humanreadable);
 
-		statsTbl.row(archive, sizeStr, coveredSizeStr,
+		statsTbl.row(archive, archStat.files(), sizeStr, coveredSizeStr,
 			boost::format("%.0lf%%") % (covered.coverage() * (double)100 / size),
 			missingSizeStr, missing.str(humanreadable));
 
 		}
 	}
 
+	if (files == 0) {
+		minPreload = minSize = 0;
+	}
+
 	std::string totalStr = sizeToString(total, humanreadable);
 
 	size_t covered = total - uncovered;
 	std::string coveredStr = sizeToString(covered, humanreadable);
 	std::string uncoveredStr = sizeToString(uncovered, humanreadable);
 
+	std::string minPreloadStr = sizeToString(minPreload, humanreadable);
+	std::string maxPreloadStr = sizeToString(maxPreload, humanreadable);
+	std::string avgPreloadStr = sizeToString(sumPreload / files, humanreadable);
+	
+	std::string minSizeStr = sizeToString(minSize, humanreadable);
+	std::string maxSizeStr = sizeToString(maxSize, humanreadable);
+	std::string avgSizeStr = sizeToString(sumSize / files, humanreadable);
+
+	ConsoleTable totalsTbl;
+	totalsTbl.columns(ConsoleTable::LEFT, ConsoleTable::RIGHT, ConsoleTable::RIGHT);
+	totalsTbl.row("Total Files:", files);
+	totalsTbl.row("Total Size", totalStr);
+	totalsTbl.row("Total Covered:", coveredStr, boost::format("%.0lf%%") % (covered * (double)100 / total));
+	totalsTbl.row("Total Missing:", uncoveredStr);
+
+	ConsoleTable sizesTbl;
+	sizesTbl.columns(ConsoleTable::LEFT, ConsoleTable::RIGHT, ConsoleTable::RIGHT, ConsoleTable::RIGHT);
+	sizesTbl.row("", "Min", "Max", "Avg");
+	sizesTbl.row("Preload Size", minPreloadStr, maxPreloadStr, avgPreloadStr);
+	sizesTbl.row("File Size", minSizeStr, maxSizeStr, avgSizeStr);
+
 	statsTbl.print(std::cout);
-
-	std::cout
-		<< (boost::format("\n"
-			"Total Size: %s\n"
-			"Total Covered: %s (%.0lf%%)\n"
-			"Total Missing: %s\n")
-		% totalStr
-		% coveredStr % (covered * (double)100 / total)
-		% uncoveredStr);
+	std::cout.put('\n');
+	totalsTbl.print(std::cout);
+	std::cout.put('\n');
+	sizesTbl.print(std::cout);
 }
 
 int main(int argc, char *argv[]) {
 		("directory,C",      po::value<std::string>(), "extract files into another directory")
 		("stop,s",           "stop on error")
 		("coverage",         "coverage analysis of archive data (archive debugging)")
+		("all,a",            "also show archives 100% coverage")
 		("dump-uncovered",   "dump uncovered areas into files (implies --coverage, archive debugging)");
 
 	po::options_description hidden;
 	bool coverage      = vm.count("coverage") > 0;
 	bool dump          = vm.count("dump-uncovered") > 0;
 	bool humanreadable = vm.count("human-readable") > 0;
+	bool printall      = vm.count("all")            > 0;
 
 	std::string directory = vm.count("directory") > 0 ? vm["directory"].as<std::string>() : std::string(".");
 	std::string archive   = vm.count("archive")   > 0 ? vm["archive"].as<std::string>()   : std::string("-");
 		}
 
 		if (coverage || dump) {
-			::coverage(package, dump, directory, humanreadable);
+			::coverage(package, dump, directory, humanreadable, printall);
 		}
 		else if (list) {
 			printListing(package, humanreadable, sorting);