From aea35fa7e1a3261e5e7a3e8daa999bed3dcd193f Mon Sep 17 00:00:00 2001 From: Till Höppner Date: Thu, 18 Feb 2016 18:53:30 +0100 Subject: Restructure, add API, add compression --- Cargo.toml | 13 ---- codegen/Cargo.toml | 14 +++++ codegen/src/lib.rs | 147 +++++++++++++++++++++++++++++++++++++++++++ example/Cargo.toml | 17 +++++ example/build.rs | 10 +++ example/data/empty | 0 example/data/foo | 1 + example/data/inner/boom | 1 + example/src/main.rs | 11 ++++ examples/foo/Cargo.toml | 13 ---- examples/foo/build.rs | 5 -- examples/foo/data/empty | 0 examples/foo/data/foo | 1 - examples/foo/data/inner/boom | 1 - examples/foo/src/main.rs | 9 --- lib/Cargo.toml | 17 +++++ lib/src/lib.rs | 65 +++++++++++++++++++ src/lib.rs | 52 --------------- 18 files changed, 283 insertions(+), 94 deletions(-) delete mode 100644 Cargo.toml create mode 100644 codegen/Cargo.toml create mode 100644 codegen/src/lib.rs create mode 100644 example/Cargo.toml create mode 100644 example/build.rs create mode 100644 example/data/empty create mode 100644 example/data/foo create mode 100644 example/data/inner/boom create mode 100644 example/src/main.rs delete mode 100644 examples/foo/Cargo.toml delete mode 100644 examples/foo/build.rs delete mode 100644 examples/foo/data/empty delete mode 100644 examples/foo/data/foo delete mode 100644 examples/foo/data/inner/boom delete mode 100644 examples/foo/src/main.rs create mode 100644 lib/Cargo.toml create mode 100644 lib/src/lib.rs delete mode 100644 src/lib.rs diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index b6bbd4c..0000000 --- a/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "includedir" -description = "Include a whole directory tree at compile time!" -keywords = ["include", "tree", "directory", "build", "static"] -version = "0.1.1" -authors = ["Till Höppner "] -license = "MIT" -homepage = "https://github.com/tilpner/includedir" -repository = "https://github.com/tilpner/includedir" - -[dependencies] -walkdir = "0.1.5" -phf_codegen = "0.7.12" diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml new file mode 100644 index 0000000..1fb1cad --- /dev/null +++ b/codegen/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "includedir_codegen" +description = "Include a whole directory tree at compile time! - Compile time part" +keywords = ["include", "tree", "directory", "build", "static"] +version = "0.1.1" +authors = ["Till Höppner "] +license = "MIT" +homepage = "https://github.com/tilpner/includedir" +repository = "https://github.com/tilpner/includedir" + +[dependencies] +walkdir = "0.1.5" +phf_codegen = "0.7.12" +flate2 = "0.2.13" diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs new file mode 100644 index 0000000..e8a6475 --- /dev/null +++ b/codegen/src/lib.rs @@ -0,0 +1,147 @@ +extern crate walkdir; +extern crate phf_codegen; +extern crate flate2; + +use std::{fmt, env, io}; +use std::borrow::{Borrow, Cow}; +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::{BufReader, BufWriter, Write}; +use std::path::{Path, PathBuf}; + +use walkdir::WalkDir; + +use flate2::FlateWriteExt; + +#[derive(Copy, Clone, Debug)] +pub enum Compression { + None, + Gzip, +} + +impl fmt::Display for Compression { + fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> { + match *self { + Compression::None => fmt.write_str("None"), + Compression::Gzip => fmt.write_str("Gzip"), + } + } +} + +pub struct IncludeDir { + files: HashMap, + name: String, +} + +pub fn start(static_name: &str) -> IncludeDir { + IncludeDir { + files: HashMap::new(), + name: static_name.to_owned(), + } +} + +#[cfg(windows)] +fn as_key(path: &str) -> Cow { + Cow::Owned(path.replace("\\", "/")) +} + +#[cfg(not(windows))] +fn as_key(path: &str) -> Cow { + Cow::Borrowed(path) +} + +impl IncludeDir { + /// Add a single file to the binary. + /// With Gzip compression, the file will be encoded to OUT_DIR first. + /// For chaining, it's not sensible to return a Result. If any to-be-included + /// files can't be found, or encoded, this function will panic!. + pub fn file>(&mut self, path: P, comp: Compression) -> &mut IncludeDir { + self.add_file(path, comp).unwrap(); + self + } + + /// ## Panics + /// + /// This function panics when CARGO_MANIFEST_DIR or OUT_DIR are not defined. + pub fn add_file>(&mut self, path: P, comp: Compression) -> io::Result<()> { + let key = path.as_ref().to_string_lossy(); + match comp { + Compression::None => { + self.files.insert(as_key(key.borrow()).into_owned(), + (comp, path.as_ref().clone().to_owned())); + } + Compression::Gzip => { + // gzip encode file to OUT_DIR + let in_path = Path::new(&env::var("CARGO_MANIFEST_DIR").unwrap()).join(&path); + let mut in_file = BufReader::new(try!(File::open(&in_path))); + + let out_path = Path::new(&env::var("OUT_DIR").unwrap()).join(&path); + try!(fs::create_dir_all(&out_path.parent().unwrap())); + let out_file = BufWriter::new(try!(File::create(&out_path))); + let mut encoder = out_file.gz_encode(flate2::Compression::Default); + + try!(io::copy(&mut in_file, &mut encoder)); + + self.files.insert(as_key(key.borrow()).into_owned(), + (comp, out_path.to_owned())); + } + } + Ok(()) + } + + /// Add a whole directory recursively to the binary. + /// This function calls `file`, and therefore will panic! on missing files. + pub fn dir>(&mut self, path: P, comp: Compression) -> &mut IncludeDir { + self.add_dir(path, comp).unwrap(); + self + } + + /// ## Panics + /// + /// This function panics when CARGO_MANIFEST_DIR or OUT_DIR are not defined. + pub fn add_dir>(&mut self, path: P, comp: Compression) -> io::Result<()> { + for entry in WalkDir::new(path).follow_links(true).into_iter() { + match entry { + Ok(ref e) if !e.file_type().is_dir() => { + try!(self.add_file(e.path(), comp)); + } + _ => (), + } + } + Ok(()) + } + + pub fn build(&self, out_name: &str) -> io::Result<()> { + let base_path = Path::new(&env::var("CARGO_MANIFEST_DIR").unwrap()).to_owned(); + let out_path = Path::new(&env::var("OUT_DIR").unwrap()).join(out_name); + let mut out_file = BufWriter::new(try!(File::create(&out_path))); + + try!(write!(&mut out_file, + "use includedir::*;\n\ + pub static {}: Files = Files {{\n\ + \tfiles: ", + self.name)); + + let entries: Vec<_> = self.files //accumulate_entries() + .iter() + .map(|(name, &(ref comp, ref path))| { + let include_path = format!("{}", + base_path.join(path).display()); + let code = format!("(Compression::{}, \ + include_bytes!(\"{}\") as &'static \ + [u8])", + comp, + as_key(&include_path)); + (as_key(&name).into_owned(), code) + }) + .collect(); + let mut map: phf_codegen::Map<&str> = phf_codegen::Map::new(); + for &(ref name, ref code) in &entries { + map.entry(&name, &code); + } + try!(map.build(&mut out_file)); + + try!(write!(&mut out_file, "\n}};\n")); + Ok(()) + } +} diff --git a/example/Cargo.toml b/example/Cargo.toml new file mode 100644 index 0000000..678f8fb --- /dev/null +++ b/example/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "foo" +version = "0.1.0" +authors = ["Till Höppner "] +build = "build.rs" +include = ["data", "data/*", "data/**"] +publish = false + +[dependencies] +phf = "0.7.12" +# includedir = "0.1.1" +includedir = { path = "../lib" } + +[build-dependencies] +# includedir = "0.1.1" +includedir = { path = "../lib" } +includedir_codegen = { path = "../codegen" } diff --git a/example/build.rs b/example/build.rs new file mode 100644 index 0000000..fb23001 --- /dev/null +++ b/example/build.rs @@ -0,0 +1,10 @@ +extern crate includedir_codegen; + +use includedir_codegen::Compression; + +fn main() { + includedir_codegen::start("FILES") + .dir("data", Compression::Gzip) + .build("data.rs") + .unwrap(); +} diff --git a/example/data/empty b/example/data/empty new file mode 100644 index 0000000..e69de29 diff --git a/example/data/foo b/example/data/foo new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/example/data/foo @@ -0,0 +1 @@ +foo diff --git a/example/data/inner/boom b/example/data/inner/boom new file mode 100644 index 0000000..9e2ba7e --- /dev/null +++ b/example/data/inner/boom @@ -0,0 +1 @@ +boom diff --git a/example/src/main.rs b/example/src/main.rs new file mode 100644 index 0000000..9ca782c --- /dev/null +++ b/example/src/main.rs @@ -0,0 +1,11 @@ +extern crate includedir; +extern crate phf; + +include!(concat!(env!("OUT_DIR"), "/data.rs")); + +fn main() { + println!("{:?}", FILES.get("data/foo")) + // for (k, v) in FILES.entries() { + // println!("{}: {} bytes", k, v.len()); + // } +} diff --git a/examples/foo/Cargo.toml b/examples/foo/Cargo.toml deleted file mode 100644 index 93c06c8..0000000 --- a/examples/foo/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "foo" -version = "0.1.0" -authors = ["Till Höppner "] -build = "build.rs" -include = ["data"] -publish = false - -[dependencies] -phf = "0.7.12" - -[build-dependencies] -includedir = "0.1.1" diff --git a/examples/foo/build.rs b/examples/foo/build.rs deleted file mode 100644 index 08c0a34..0000000 --- a/examples/foo/build.rs +++ /dev/null @@ -1,5 +0,0 @@ -extern crate includedir; - -fn main() { - includedir::build("data").unwrap(); -} diff --git a/examples/foo/data/empty b/examples/foo/data/empty deleted file mode 100644 index e69de29..0000000 diff --git a/examples/foo/data/foo b/examples/foo/data/foo deleted file mode 100644 index 257cc56..0000000 --- a/examples/foo/data/foo +++ /dev/null @@ -1 +0,0 @@ -foo diff --git a/examples/foo/data/inner/boom b/examples/foo/data/inner/boom deleted file mode 100644 index 9e2ba7e..0000000 --- a/examples/foo/data/inner/boom +++ /dev/null @@ -1 +0,0 @@ -boom diff --git a/examples/foo/src/main.rs b/examples/foo/src/main.rs deleted file mode 100644 index 8270aec..0000000 --- a/examples/foo/src/main.rs +++ /dev/null @@ -1,9 +0,0 @@ -extern crate phf; - -include!(concat!(env!("OUT_DIR"), "/dir_data.rs")); - -fn main() { - for (k, v) in FILES.entries() { - println!("{}: {} bytes", k, v.len()); - } -} diff --git a/lib/Cargo.toml b/lib/Cargo.toml new file mode 100644 index 0000000..aa82ad8 --- /dev/null +++ b/lib/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "includedir" +description = "Include a whole directory tree at compile time! - Runtime part" +keywords = ["include", "tree", "directory", "build", "static"] +version = "0.1.1" +authors = ["Till Höppner "] +license = "MIT" +homepage = "https://github.com/tilpner/includedir" +repository = "https://github.com/tilpner/includedir" + +[features] +default = ["flate2"] + +[dependencies] +phf = "0.7.12" + +flate2 = { version = "*", optional = true } diff --git a/lib/src/lib.rs b/lib/src/lib.rs new file mode 100644 index 0000000..666c727 --- /dev/null +++ b/lib/src/lib.rs @@ -0,0 +1,65 @@ +extern crate phf; +extern crate flate2; + +use std::borrow::{Borrow, Cow}; +use std::io::{self, Cursor, Error, ErrorKind, Read}; + +use flate2::FlateReadExt; + +pub enum Compression { + None, + Gzip, +} + +/// Runtime access to the included files +pub struct Files { + /// **Do not access this field, it is only public to allow for code generation!** + pub files: phf::Map<&'static str, (Compression, &'static [u8])>, +} + +#[cfg(windows)] +fn as_key(path: &str) -> Cow { + Cow::Owned(path.replace("\\", "/")) +} + +#[cfg(not(windows))] +fn as_key(path: &str) -> Cow { + Cow::Borrowed(path) +} + +impl Files { + pub fn available(&self, path: &str) -> bool { + self.files.contains_key(path) + } + + pub fn get(&self, path: &str) -> io::Result> { + let key = as_key(path); + match self.files.get(key.borrow() as &str) { + Some(b) => { + match b.0 { + Compression::None => Ok(Cow::Borrowed(b.1)), + Compression::Gzip => { + let mut r = try!(Cursor::new(b.1).gz_decode()); + let mut v = Vec::new(); + try!(r.read_to_end(&mut v)); + Ok(Cow::Owned(v)) + } + } + } + None => Err(Error::new(ErrorKind::NotFound, "Key not found")), + } + } + + pub fn read(&self, path: &str) -> io::Result> { + let key = as_key(path); + match self.files.get(key.borrow() as &str) { + Some(b) => { + match b.0 { + Compression::None => Ok(Box::new(Cursor::new(b.1))), + Compression::Gzip => Ok(Box::new(try!(Cursor::new(b.1).gz_decode()))), + } + } + None => Err(Error::new(ErrorKind::NotFound, "Key not found")), + } + } +} diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 939b958..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,52 +0,0 @@ -extern crate walkdir; -extern crate phf_codegen; - -use std::{env, io}; -use std::fs::File; -use std::io::{BufWriter, Write}; -use std::path::Path; - -use walkdir::WalkDir; - -pub fn build>(dir: P) -> io::Result<()> { - let dir = dir.as_ref(); - - let base_path = Path::new(&env::var("CARGO_MANIFEST_DIR").unwrap()).to_owned(); - - let dir_name = dir.file_name().expect("Invalid directory name").to_string_lossy(); - let out_name = format!("dir_{}.rs", dir_name); - let out_path = Path::new(&env::var("OUT_DIR").unwrap()).join(out_name); - let mut out_file = BufWriter::new(File::create(&out_path).unwrap()); - - write!(&mut out_file, - "static FILES: phf::Map<&'static str, &'static [u8]> = ") - .unwrap(); - - // FIXME: rustfmt mangles this - let entries: Vec<(String, String)> = WalkDir::new(dir) - .into_iter() - .filter(|e| { - !e.as_ref() - .ok() - .map_or(true, |e| e.file_type().is_dir()) - }) - .map(|e| { - let entry = e.expect("Invalid dir entry."); - let name = format!("{}", entry.path().display()); - let code = format!("include_bytes!(\"{}\") \ - as &'static [u8]", - base_path.join(entry.path()) - .display()); - (name, code) - }) - .collect(); - - let mut map: phf_codegen::Map<&str> = phf_codegen::Map::new(); - for &(ref name, ref code) in &entries { - map.entry(name, code); - } - map.build(&mut out_file).unwrap(); - - write!(&mut out_file, ";\n").unwrap(); - Ok(()) -} -- cgit v1.2.3