master
q3k 2021-04-03 13:05:13 +02:00
parent d8e965b0b4
commit a1740ebbec
20 changed files with 1209 additions and 90 deletions

193
Cargo.lock generated
View File

@ -38,12 +38,45 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "bigbrain"
version = "0.1.0"
dependencies = [
"byteorder",
"flate2",
"getrandom",
"image",
"itertools",
"log",
"protobuf",
"rand",
"rand_distr",
"rayon",
]
[[package]]
name = "bigbrainjs"
version = "0.1.0"
dependencies = [
"bigbrain",
"console_error_panic_hook",
"protobuf",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "bitflags"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "bumpalo"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
[[package]]
name = "bytemuck"
version = "1.5.1"
@ -56,6 +89,12 @@ version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -68,13 +107,23 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "console_error_panic_hook"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211"
dependencies = [
"cfg-if 0.1.10",
"wasm-bindgen",
]
[[package]]
name = "crc32fast"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -83,7 +132,7 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"crossbeam-utils",
]
@ -93,7 +142,7 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils",
]
@ -104,7 +153,7 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"crossbeam-utils",
"lazy_static",
"memoffset",
@ -118,7 +167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
dependencies = [
"autocfg",
"cfg-if",
"cfg-if 1.0.0",
"lazy_static",
]
@ -157,7 +206,7 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"crc32fast",
"libc",
"miniz_oxide 0.4.4",
@ -169,9 +218,11 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"js-sys",
"libc",
"wasi",
"wasm-bindgen",
]
[[package]]
@ -227,12 +278,6 @@ dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
[[package]]
name = "jpeg-decoder"
version = "0.1.22"
@ -242,6 +287,15 @@ dependencies = [
"rayon",
]
[[package]]
name = "js-sys"
version = "0.3.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
@ -266,7 +320,7 @@ version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -307,18 +361,10 @@ dependencies = [
name = "neural"
version = "0.1.0"
dependencies = [
"byteorder",
"bigbrain",
"env_logger",
"flate2",
"image",
"itertools",
"log",
"protobuf",
"rand",
"rand_distr",
"rayon",
"serde",
"serde_json",
]
[[package]]
@ -507,12 +553,6 @@ version = "0.6.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
[[package]]
name = "ryu"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "scoped_threadpool"
version = "0.1.9"
@ -525,37 +565,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "1.0.67"
@ -599,6 +608,70 @@ version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9"
dependencies = [
"cfg-if 1.0.0",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489"
[[package]]
name = "web-sys"
version = "0.3.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "weezl"
version = "0.1.4"

View File

@ -6,16 +6,15 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[workspace]
members = [
"bigbrain",
"bigbrainjs",
]
[dependencies]
byteorder = "1"
flate2 = "1"
bigbrain = { path = "./bigbrain" }
log = "0.4.0"
env_logger = "0.8.3"
rand = "0.8.3"
rand_distr = "0.4.0"
rayon = "1"
itertools = "0.10.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
image = "0.23"
protobuf = "2"

1
bigbrain/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
target

651
bigbrain/Cargo.lock generated Normal file
View File

@ -0,0 +1,651 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "adler32"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "aho-corasick"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
dependencies = [
"memchr",
]
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "bigbrain"
version = "0.1.0"
dependencies = [
"env_logger",
"getrandom",
"image",
"itertools",
"log",
"protobuf",
"rand",
"rand_distr",
"rayon",
"wasm-bindgen",
]
[[package]]
name = "bitflags"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "bumpalo"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
[[package]]
name = "bytemuck"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "crc32fast"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
dependencies = [
"cfg-if",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
dependencies = [
"autocfg",
"cfg-if",
"lazy_static",
]
[[package]]
name = "deflate"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174"
dependencies = [
"adler32",
"byteorder",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "env_logger"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f"
dependencies = [
"atty",
"humantime",
"log",
"regex",
"termcolor",
]
[[package]]
name = "getrandom"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi",
"wasm-bindgen",
]
[[package]]
name = "gif"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a668f699973d0f573d15749b7002a9ac9e1f9c6b220e7b165601334c173d8de"
dependencies = [
"color_quant",
"weezl",
]
[[package]]
name = "hermit-abi"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
dependencies = [
"libc",
]
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "image"
version = "0.23.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"gif",
"jpeg-decoder",
"num-iter",
"num-rational",
"num-traits",
"png",
"scoped_threadpool",
"tiff",
]
[[package]]
name = "itertools"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319"
dependencies = [
"either",
]
[[package]]
name = "jpeg-decoder"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2"
dependencies = [
"rayon",
]
[[package]]
name = "js-sys"
version = "0.3.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56d855069fafbb9b344c0f962150cd2c1187975cb1c22c1522c240d8c4986714"
[[package]]
name = "libm"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a"
[[package]]
name = "log"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
]
[[package]]
name = "memchr"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memoffset"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d"
dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435"
dependencies = [
"adler32",
]
[[package]]
name = "miniz_oxide"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
dependencies = [
"adler",
"autocfg",
]
[[package]]
name = "num-integer"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg",
"libm",
]
[[package]]
name = "num_cpus"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "png"
version = "0.16.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6"
dependencies = [
"bitflags",
"crc32fast",
"deflate",
"miniz_oxide 0.3.7",
]
[[package]]
name = "ppv-lite86"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "proc-macro2"
version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec"
dependencies = [
"unicode-xid",
]
[[package]]
name = "protobuf"
version = "2.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b7f4a129bb3754c25a4e04032a90173c68f85168f77118ac4cb4936e7f06f92"
[[package]]
name = "quote"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
"rand_hc",
]
[[package]]
name = "rand_chacha"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
dependencies = [
"getrandom",
]
[[package]]
name = "rand_distr"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da9e8f32ad24fb80d07d2323a9a2ce8b30d68a62b8cb4df88119ff49a698f038"
dependencies = [
"num-traits",
"rand",
]
[[package]]
name = "rand_hc"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
dependencies = [
"rand_core",
]
[[package]]
name = "rayon"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
[[package]]
name = "regex"
version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
[[package]]
name = "scoped_threadpool"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "syn"
version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ce15dd3ed8aa2f8eeac4716d6ef5ab58b6b9256db41d7e1a0224c2788e8fd87"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "termcolor"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
dependencies = [
"winapi-util",
]
[[package]]
name = "tiff"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437"
dependencies = [
"jpeg-decoder",
"miniz_oxide 0.4.4",
"weezl",
]
[[package]]
name = "unicode-xid"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489"
[[package]]
name = "weezl"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a32b378380f4e9869b22f0b5177c68a5519f03b3454fde0b291455ddbae266c"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

19
bigbrain/Cargo.toml Normal file
View File

@ -0,0 +1,19 @@
[package]
name = "bigbrain"
version = "0.1.0"
authors = ["Serge Bazanski <q3k@q3k.org>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
byteorder = "1"
flate2 = "1"
log = "0.4.0"
getrandom = { version = "0.2.2", features = ["js"] }
rand = "0.8.3"
rand_distr = "0.4.0"
rayon = "1"
itertools = "0.10.0"
image = "0.23"
protobuf = "2"

5
bigbrain/src/lib.rs Normal file
View File

@ -0,0 +1,5 @@
pub mod maths;
pub mod mnist;
pub mod network;
pub mod proto;
pub mod types;

View File

@ -45,6 +45,17 @@ impl Image {
})
}
pub fn from_vec(data: &Vec<f32>) -> Self {
if data.len() != 28*28 {
panic!("invalid data len");
}
Self {
pixels: data.iter().cloned().collect(),
width: 28,
height: 28,
}
}
pub fn to_rgb(
&self,
) -> RgbImage {

View File

@ -6,6 +6,13 @@ use crate::types::{Data, Input, Output};
use crate::proto::network as npb;
use crate::maths::{Vector, Matrix, Shape};
#[derive(Debug)]
pub enum Error {
WrongProto(String),
}
pub type Result<T> = std::result::Result<T, Error>;
pub struct Network<D: Data> {
sizes: Vec<usize>,
biases: Vec<Vector>,
@ -29,6 +36,39 @@ impl<D: Data> Network<D> {
}
}
pub fn from_proto(n: &npb::Network) -> Result<Self> {
let mut biases: Vec<Vector> = vec![];
let mut weights: Vec<Matrix> = vec![];
let mut sizes: Vec<usize> = vec![];
if n.get_input().size <= 0 {
return Err(Error::WrongProto("input layer must have size".into()));
}
let mut prev_size = n.get_input().size as usize;
sizes.push(prev_size);
for layer in &n.inner {
let rows = layer.biases.len();
let columns = layer.weights.len() / rows;
if columns != prev_size {
return Err(Error::WrongProto(format!("size mismatch, {} columns, {} prev", columns, prev_size)));
}
biases.push(Vector(layer.biases.clone()));
let mut rowvec: Vec<Vector> = Vec::with_capacity(rows);
for nr in 0..rows {
rowvec.push(layer.weights[nr*columns..(nr+1)*columns].iter().cloned().collect());
}
weights.push(Matrix::new(rowvec));
prev_size = rows;
sizes.push(prev_size);
}
Ok(Self {
sizes, weights, biases,
phantom: std::marker::PhantomData,
})
}
pub fn proto(&self) -> npb::Network {
let mut res = npb::Network::new();
@ -66,7 +106,7 @@ impl<D: Data> Network<D> {
}
}
fn feedforward(&self, i: &D::Input) -> D::Output {
pub fn feedforward(&self, i: &D::Input) -> D::Output {
let mut a = i.data().clone();
for (b, w) in self.biases.iter().zip(self.weights.iter()) {
let mut z = w.mult_vec(&a);
@ -145,7 +185,7 @@ impl<D: Data> Network<D> {
}
}
fn evaluate(
pub fn evaluate(
&self,
test_data: &D,
) -> usize {

2
bigbrainjs/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
target
pkg

124
bigbrainjs/Cargo.lock generated Normal file
View File

@ -0,0 +1,124 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "bigbrainjs"
version = "0.1.0"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "bumpalo"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "log"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
]
[[package]]
name = "proc-macro2"
version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec"
dependencies = [
"unicode-xid",
]
[[package]]
name = "quote"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "syn"
version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ce15dd3ed8aa2f8eeac4716d6ef5ab58b6b9256db41d7e1a0224c2788e8fd87"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "unicode-xid"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "wasm-bindgen"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489"

23
bigbrainjs/Cargo.toml Normal file
View File

@ -0,0 +1,23 @@
[package]
name = "bigbrainjs"
version = "0.1.0"
authors = ["Serge Bazanski <q3k@q3k.org>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
crate-type = ["cdylib"]
[dependencies]
bigbrain = { path = "../bigbrain" }
wasm-bindgen = "0.2"
protobuf = "2"
console_error_panic_hook = "0.1"
[dependencies.web-sys]
version = "0.3"
features = [
"console",
]

42
bigbrainjs/src/lib.rs Normal file
View File

@ -0,0 +1,42 @@
use wasm_bindgen::prelude::*;
use protobuf::Message;
use bigbrain::types::Output;
macro_rules! log {
( $( $t:tt )* ) => {
web_sys::console::log_1(&format!( $( $t )* ).into());
}
}
use bigbrain::{
network,
mnist,
proto::network as npb,
};
#[wasm_bindgen]
pub struct Brain {
net: network::Network<mnist::Data>,
}
#[wasm_bindgen]
impl Brain {
pub fn new(pb: Box<[u8]>) -> Self {
console_error_panic_hook::set_once();
let pb = npb::Network::parse_from_bytes(&pb).unwrap();
Brain {
net: network::Network::from_proto(&pb).unwrap(),
}
}
pub fn see(&self, data: Box<[u8]>) -> Vec<f32> {
let vec: Vec<f32> = data.iter().map(|el| {
((255 - el) as f32) / 255.0
}).collect();
//log!("{:?}", vec);
let img = mnist::Image::from_vec(&vec);
let res = self.net.feedforward(&img);
return res.data().iter().cloned().collect();
}
}

35
bigbrainjs/web/main.html Normal file
View File

@ -0,0 +1,35 @@
<!doctype HTML>
<html>
<head>
<meta charset="UTF-8">
<title>bigbrain.js</title>
<style>
#digit0, #digit1, #digit2, #digit3, #digit4,
#digit5, #digit6, #digit7, #digit8, #digit9 {
background-color: red;
width: 20px;
height: 80px;
float: left;
margin-right: 5px;
color: white;
}
</style>
</head>
<body>
<canvas id="myCanvas" width="200" height="200" style="border: 1px solid black;"></canvas>
<div id="histogram">
<div id="digit0"> 0 </div>
<div id="digit1"> 1 </div>
<div id="digit2"> 2 </div>
<div id="digit3"> 3 </div>
<div id="digit4"> 4 </div>
<div id="digit5"> 5 </div>
<div id="digit6"> 6 </div>
<div id="digit7"> 7 </div>
<div id="digit8"> 8 </div>
<div id="digit9"> 9 </div>
</div>
</body>
<script type="module" src="main.js"></script>
</html>

92
bigbrainjs/web/main.js Normal file
View File

@ -0,0 +1,92 @@
import { Brain, default as init } from './pkg/bigbrainjs.js';
let canvas = null;
let ctx = null;
let pos = { x: 0, y: 0 };
let brain = null;
document.addEventListener('DOMContentLoaded', async () => {
await init();
let response = await fetch('/net.pb');
let blob = await response.blob(); // download as Blob object
let data = new Uint8Array(await blob.arrayBuffer());
brain = Brain.new(data);
console.log(brain);
canvas = document.getElementById("myCanvas");
ctx = canvas.getContext('2d');
const w = canvas.getBoundingClientRect().width;
const h = canvas.getBoundingClientRect().height;
ctx.rect(0, 0, w, h);
ctx.fillStyle = "white";
ctx.fill();
});
const pokeBrain = () => {
const w = canvas.getBoundingClientRect().width;
const h = canvas.getBoundingClientRect().height;
let data = ctx.getImageData(0, 0, w, h);
let sx = w / 28;
let sy = h / 28;
let mnist = new Uint8Array(28*28);
for (let x = 0; x < 28; x++) {
for (let y = 0; y < 28; y++) {
let cx = Math.floor(x * sx);
let cy = Math.floor(y * sy);
let ix = (cx + cy * w) * 4;
//let ix = x + y * w * 4;
//console.log(data.data[Math.floor(x*sx + y*sy*w)]);
//console.log(data.data[ix]);
mnist[x+y*28] = data.data[ix];
}
}
//for (let y = 0; y < 28; y++) {
// let line = Array.from(mnist.subarray(y*28, (y+1)*28)).map((el) => {
// if (el == 0) {
// return "#";
// }
// return ".";
// });
// console.log(y, line.join(""));
//}
//console.log("===");
let res = brain.see(mnist);
//console.log(res);
let i = 0;
for (let digit of res) {
let val = digit * 100 + 20 + "px";
document.querySelector("#digit"+i).style.height = val;
i += 1;
}
};
document.addEventListener('mousemove', (e) => {
if (e.buttons !== 1) return;
ctx.beginPath(); // begin
ctx.lineWidth = 16;
ctx.lineCap = 'round';
ctx.strokeStyle = '#000000';
ctx.moveTo(pos.x, pos.y); // from
pos.x = e.clientX;
pos.y = e.clientY;
ctx.lineTo(pos.x, pos.y); // to
ctx.stroke(); // draw it!
pokeBrain();
});
document.addEventListener('mousedown', (e) => {
pos.x = e.clientX;
pos.y = e.clientY;
});
document.addEventListener('mouseenter', (e) => {
pos.x = e.clientX;
pos.y = e.clientY;
});

1
bigbrainjs/web/net.pb Symbolic link
View File

@ -0,0 +1 @@
../../net.pb

View File

@ -1,10 +1,6 @@
use std::time;
pub mod types;
pub mod maths;
pub mod proto;
pub mod network;
mod mnist;
use bigbrain::{mnist,network,proto::network as npb};
use protobuf::Message;
@ -22,18 +18,23 @@ fn main() {
log::info!("Split into {} training and {} validation", num_train, num_validation);
let (train, validation) = train.split(num_train);
let now = time::Instant::now();
let size = vec![
28*28 as usize,
30usize,
10usize,
];
let mut net = network::Network::new(&size);
let elapsed = now.elapsed();
log::info!("Created random network {:?} in {:?}.", size, elapsed);
net.sgd(&train, 30, 32, 3.0, Some(&test));
let proto = net.proto();
let mut f = std::fs::File::create("net.pb").unwrap();
proto.write_to_writer(&mut f).unwrap();
//let now = time::Instant::now();
//let size = vec![
// 28*28 as usize,
// 30usize,
// 10usize,
//];
//let mut net = network::Network::new(&size);
//let elapsed = now.elapsed();
//log::info!("Created random network {:?} in {:?}.", size, elapsed);
//net.sgd(&train, 30, 32, 3.0, Some(&test));
//let proto = net.proto();
//let mut f = std::fs::File::create("net.pb").unwrap();
//proto.write_to_writer(&mut f).unwrap();
let nbytes = std::fs::read("net.pb").unwrap();
let n = npb::Network::parse_from_bytes(&nbytes).unwrap();
let net = network::Network::from_proto(&n).unwrap();
let res = net.evaluate(&test);
log::info!("{}", res);
}