~saiko/modsite

32af7abe3769f6a4912dda1d4708504d87a61949 — 2xsaiko a month ago e1551b9 master
did a thing
M Cargo.lock => Cargo.lock +35 -1
@@ 591,7 591,7 @@ dependencies = [
[[package]]
name = "cmdparser"
version = "0.1.0"
source = "git+https://git.dblsaiko.net/cmdparser.git#1c1e5a9d4aef395ffed336d0b723af2613fb4d59"
source = "git+https://git.2x.ax/~saiko/cmdparser#1c1e5a9d4aef395ffed336d0b723af2613fb4d59"

[[package]]
name = "copyless"


@@ 656,6 656,16 @@ dependencies = [
]

[[package]]
name = "deflate"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e5d2a2273fed52a7f947ee55b092c4057025d7a3e04e5ecdbd25d6c3fb1bd7"
dependencies = [
 "adler32",
 "byteorder",
]

[[package]]
name = "derive_more"
version = "0.99.7"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1085,6 1095,15 @@ dependencies = [
]

[[package]]
name = "inflate"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
dependencies = [
 "adler32",
]

[[package]]
name = "iovec"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1317,15 1336,18 @@ dependencies = [
 "cmdparser",
 "futures-util",
 "handlebars",
 "hex",
 "itertools",
 "log",
 "num_cpus",
 "png",
 "reqwest",
 "semver_rs",
 "serde",
 "serde-xml-rs",
 "serde_json",
 "serde_urlencoded",
 "sha2",
 "simplelog",
 "sqlx",
 "tokio",


@@ 1572,6 1594,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"

[[package]]
name = "png"
version = "0.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c68a431ed29933a4eb5709aca9800989758c97759345860fa5db3cfced0b65d"
dependencies = [
 "bitflags",
 "crc32fast",
 "deflate",
 "inflate",
]

[[package]]
name = "podio"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"

M migrations/2020-05-06-184920_file_version_passthrough/apply.sql => migrations/2020-05-06-184920_file_version_passthrough/apply.sql +1 -1
@@ 1,5 1,5 @@
ALTER TABLE mod_base
    DROP CONSTRAINT mod_base_uuid_display_version_fkey;
    DROP CONSTRAINT mod_base_uuid_fkey;

ALTER TABLE mod_version
    ADD file_version varchar;

M migrations/2020-05-06-184920_file_version_passthrough/unapply.sql => migrations/2020-05-06-184920_file_version_passthrough/unapply.sql +1 -1
@@ 1,5 1,5 @@
ALTER TABLE mod_base
DROP CONSTRAINT mod_base_uuid_display_version_fkey;
DROP CONSTRAINT mod_base_uuid_fkey;

ALTER TABLE mod_version
    DROP CONSTRAINT mod_version_pkey;

A migrations/20200525112146-add-mod-icon-field/_props => migrations/20200525112146-add-mod-icon-field/_props +4 -0
@@ 0,0 1,4 @@
// Auto-generated migration metadata. Do not edit.
id   7a99fd89e6aa4ff9a053676976dc743b
name "Add mod icon field"
date 1590405706

A migrations/20200525112146-add-mod-icon-field/apply.sql => migrations/20200525112146-add-mod-icon-field/apply.sql +12 -0
@@ 0,0 1,12 @@
CREATE TABLE icon
(
    uuid       uuid    NOT NULL,
    input_hash bytea   NOT NULL,
    path       varchar NOT NULL,

    PRIMARY KEY (uuid),
    UNIQUE (input_hash)
);

ALTER TABLE mod_version
    ADD COLUMN icon uuid REFERENCES icon (uuid);
\ No newline at end of file

A migrations/20200525112146-add-mod-icon-field/unapply.sql => migrations/20200525112146-add-mod-icon-field/unapply.sql +4 -0
@@ 0,0 1,4 @@
ALTER TABLE mod_version
    DROP COLUMN icon;

DROP TABLE icon;
\ No newline at end of file

M migtool/Cargo.toml => migtool/Cargo.toml +2 -2
@@ 7,8 7,8 @@ edition = "2018"
[dependencies]
clap = "3.0.0-beta.1"
chrono = "0.4.11"
cmdparser = { git = "https://git.dblsaiko.net/cmdparser.git", default-features = false }
cmdparser = { git = "https://git.2x.ax/~saiko/cmdparser", default-features = false }
uuid = { version = "0.8.1", features = ["v4"] }
sqlx = { version = "0.3.5", default-features = false, features = ["runtime-tokio", "macros", "postgres", "uuid", "chrono"] }
tokio = { version =  "0.2.20", features = ["stream"] }
anyhow = "1.0.31"
\ No newline at end of file
anyhow = "1.0.31"

M site/Cargo.toml => site/Cargo.toml +5 -2
@@ 24,11 24,14 @@ handlebars = { version = "3.1.0-beta.2", features = ["dir_source"] }
futures-util = "0.3.4"
serde_urlencoded = "0.6.1"
num_cpus = "1.13.0"
cmdparser = { git = "https://git.dblsaiko.net/cmdparser.git", default-features = false }
cmdparser = { git = "https://git.2x.ax/~saiko/cmdparser", default-features = false }
log = "0.4.8"
simplelog = "0.8.0"
itertools = "0.9.0"
url = { version = "*", features = ["serde"] }
png = "0.16.3"
sha2 = "0.8.2"
hex = "0.4.2"

[build-dependencies]
cmdparser = { git = "https://git.dblsaiko.net/cmdparser.git", default-features = false }
cmdparser = { git = "https://git.2x.ax/~saiko/cmdparser", default-features = false }

M site/src/indexer/db.rs => site/src/indexer/db.rs +1 -0
@@ 21,6 21,7 @@ pub struct ModVersionInfo {
    pub license: Option<String>,
    pub environment: Environment,
    pub file_url: String,
    pub icon: Option<Vec<u8>>,

    pub links: Vec<ModLink>,
}

M site/src/indexer/mod.rs => site/src/indexer/mod.rs +16 -1
@@ 1,5 1,5 @@
use std::collections::HashMap;
use std::io::Cursor;
use std::io::{Cursor, Read};

use zip::ZipArchive;



@@ 78,6 78,20 @@ async fn index_mod(id: String, file_source: &FileSource, client: &reqwest::Clien
            }
        };

        let icon = fm.icon.and_then(|i| {
            let mut file = arc.by_name(&i).ok()?;
            let mut buf = Vec::with_capacity(file.size() as usize);
            let len = file.read_to_end(&mut buf);
            if let Err(e) = &len {
                eprintln!("error reading icon for mod {} {}: {}", id, v.version, e);
            }
            len.ok()?;
            // verify that what we're saving here actually is a png file
            // TODO: scale/size limit!
            png::Decoder::new(Cursor::new(&mut buf)).read_info().ok()?;
            Some(buf)
        });

        let mut links = Vec::new();

        for (child, v) in fm.breaks {


@@ 113,6 127,7 @@ async fn index_mod(id: String, file_source: &FileSource, client: &reqwest::Clien
            license: fm.license,
            environment: fm.environment.into(),
            file_url: v.file_url,
            icon,
            links,
        };


M site/src/main.rs => site/src/main.rs +32 -4
@@ 3,14 3,17 @@
use std::collections::HashMap;
use std::fs::File;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::time::Duration;

use hex::ToHex;
use log::info;
use log::LevelFilter;
use sha2::{Digest, Sha256};
use simplelog::{CombinedLogger, Config, ConfigBuilder, TerminalMode, TermLogger, WriteLogger};
use sqlx::{PgConnection, PgPool, Pool};
use tokio::fs;
use uuid::Uuid;

use crate::indexer::db::ModInfo;


@@ 127,10 130,14 @@ async fn put_into_db(modinfo: HashMap<String, ModInfo>, db: &Pool<PgConnection>)
        sqlx::query!("INSERT INTO mod_base (uuid, id, display_version) VALUES ($1, $2, $3)", mod_base, mod_info.id, None::<&str>).execute(&mut ta).await?;

        for (file_version, vi) in mod_info.versions {
            let icon = if let Some(icon) = &vi.icon {
                Some(create_icon(icon, db).await?)
            } else { None };

            sqlx::query_unchecked!(
                "INSERT INTO mod_version (mod_base, version, id, name, description, homepage, issues, source, license, download_link, environment, file_version) \
                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
                mod_base, vi.version, vi.modid, vi.name, vi.description, vi.homepage, vi.issues, vi.source, vi.license, vi.file_url, vi.environment, file_version)
                "INSERT INTO mod_version (mod_base, version, id, name, description, homepage, issues, source, license, download_link, environment, file_version, icon) \
                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
                mod_base, vi.version, vi.modid, vi.name, vi.description, vi.homepage, vi.issues, vi.source, vi.license, vi.file_url, vi.environment, file_version, icon)
                .execute(&mut ta).await?;

            for link in vi.links {


@@ 158,4 165,25 @@ async fn put_into_db(modinfo: HashMap<String, ModInfo>, db: &Pool<PgConnection>)

    ta.commit().await?;
    Ok(())
}

async fn create_icon(data: &[u8], db: &Pool<PgConnection>) -> anyhow::Result<Uuid> {
    let checksum = Sha256::digest(&data).to_vec();

    let icon: Option<Uuid> = sqlx::query!("SELECT uuid FROM icon WHERE input_hash = $1", checksum).fetch_optional(db).await?.map(|r| r.uuid);

    if let Some(icon) = icon {
        Ok(icon)
    } else {
        let id = Uuid::new_v4();
        let hex: String = checksum.encode_hex_upper();
        let mut pb = PathBuf::new();
        pb.push("assets");
        pb.push(&hex[..2]);
        fs::create_dir_all(&pb).await?;
        pb.push(hex);
        fs::write(&pb, &data).await?;
        sqlx::query!("INSERT INTO icon (uuid, input_hash, path) VALUES ($1, $2, $3)", id, checksum, pb.to_str().unwrap()).execute(db).await?;
        Ok(id)
    }
}
\ No newline at end of file

M site/src/site/main_page.rs => site/src/site/main_page.rs +4 -4
@@ 3,15 3,15 @@ use futures_util::TryStreamExt;
use handlebars::Handlebars;
use serde::{Deserialize, Serialize};
use sqlx::{FromRow, PgConnection, Pool};
use url::Url;

use crate::db::types::Environment;
use crate::site::db_access::{Db, map_db_err};
use crate::site::geninfo::GenInfo;

use super::pagestates::{MainPage as PageState, self};
use url::Url;
use super::pagestates::{self, MainPage as PageState};

pub async fn main_page(req: HttpRequest, page_state: web::Query<PageState<'_>>, hb: web::Data<Handlebars<'_>>, db: Db) -> Result<impl Responder, actix_web::Error> {
pub async fn main_page(req: HttpRequest, page_state: web::Query<PageState<'_>>, hb: web::Data<Handlebars<'_>>, db: Db) -> actix_web::Result<impl Responder> {
    let sorting = SortLinks::from(&page_state, &req);

    let body = hb.render("index", &Data {


@@ 91,7 91,7 @@ struct FromDb {
impl ModData {
    async fn get_all(db: &Pool<PgConnection>, req: &HttpRequest) -> sqlx::Result<Vec<ModData>> {
        let from_db: Vec<FromDb> = sqlx::query_as_unchecked!(FromDb,
                "SELECT b.id, v.id as modid, v.name, v.description, v.homepage, v.issues, v.source, v.license, v.download_link, v.environment \
                "SELECT b.ida, v.id as modid, v.name, v.description, v.homepage, v.issues, v.source, v.license, v.download_link, v.environment \
                 FROM mod_base b \
                 INNER JOIN mod_version v ON (v.file_version = b.display_version AND v.mod_base = b.uuid)")
            .fetch(db)

M site/src/site/mod.rs => site/src/site/mod.rs +2 -0
@@ 12,6 12,7 @@ use crate::site::db_access::Db;
mod geninfo;
mod main_page;
mod mod_page;
mod mod_icon;

mod pagestates;



@@ 32,6 33,7 @@ pub async fn start_site(config: &LaunchConfig, db_pool: Pool<PgConnection>) -> i
        App::new()
            .service(web::resource("/").name("main_page").guard(guard::Get()).to(main_page::main_page))
            .service(web::resource("/mod").name("mod_page").guard(guard::Get()).to(mod_page::mod_page))
            .service(web::resource("/mod/icon").name("mod_icon").guard(guard::Get()).to(mod_icon::mod_icon))
            .service(actix_files::Files::new("/static", "static"))
            .app_data(Db::new(db_pool.clone()))
            .app_data(hb.clone())

A site/src/site/mod_icon.rs => site/src/site/mod_icon.rs +32 -0
@@ 0,0 1,32 @@
use std::fs;

use actix_web::{HttpRequest, HttpResponse, Responder, web};

use crate::site::db_access::{Db, map_db_err};

use super::pagestates::ModIconPage as PageState;

pub async fn mod_icon(req: HttpRequest, state: web::Query<PageState<'_>>, db: Db) -> actix_web::Result<impl Responder> {
    let path: Option<String> = if let Some(ver) = &state.v {
        sqlx::query!(
        "SELECT i.path FROM mod_version v \
         JOIN mod_base b ON b.uuid = v.mod_base \
         JOIN icon i ON i.uuid = v.icon \
         WHERE b.id = $1 AND v.file_version = $2", state.id.as_ref(), ver.as_ref())
            .fetch_optional(&*db).await.map_err(map_db_err)?.map(|a| a.path)
    } else {
        sqlx::query!(
        "SELECT i.path FROM mod_version v \
         JOIN mod_base b ON b.uuid = v.mod_base \
         JOIN icon i ON i.uuid = v.icon \
         WHERE b.id = $1 AND v.file_version = b.display_version", state.id.as_ref())
            .fetch_optional(&*db).await.map_err(map_db_err)?.map(|a| a.path)
    };

    if let Some(icon) = path {
        let data = fs::read(icon)?;
        Ok(HttpResponse::Ok().content_type("image/png").body(data))
    } else {
        Ok(HttpResponse::Found().header("Location", "/static/icon.png").finish())
    }
}
\ No newline at end of file

M site/src/site/mod_page.rs => site/src/site/mod_page.rs +1 -2
@@ 8,13 8,12 @@ use sqlx::{FromRow, PgConnection, Pool};
use url::Url;

use crate::db::types::{Environment, LinkType};
use crate::fabric_mod::VersionRange;
use crate::site::db_access::{Db, map_db_err};
use crate::site::geninfo::GenInfo;

use super::pagestates::ModPage as PageState;

pub async fn mod_page(req: HttpRequest, web::Query(PageState { id, v: version }): web::Query<PageState<'_>>, hb: web::Data<Handlebars<'_>>, db: Db) -> Result<impl Responder, actix_web::Error> {
pub async fn mod_page(req: HttpRequest, web::Query(PageState { id, v: version }): web::Query<PageState<'_>>, hb: web::Data<Handlebars<'_>>, db: Db) -> actix_web::Result<impl Responder> {
    let from_db = if let Some(version) = version {
        FromDb::load_ver(&id, &version, &db).await.map_err(map_db_err)?
    } else {

M site/src/site/pagestates.rs => site/src/site/pagestates.rs +22 -0
@@ 45,4 45,26 @@ impl<'a> ModPage<'a> {
        url.set_query(Some(&serde_urlencoded::to_string(self).unwrap()));
        url
    }
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ModIconPage<'a> {
    pub id: Cow<'a, str>,
    pub v: Option<Cow<'a, str>>,
}

impl<'a> ModIconPage<'a> {
    pub fn from_id(id: impl Into<Cow<'a, str>>) -> Self {
        ModIconPage { id: id.into(), v: None }
    }

    pub fn from_id_ver(id: impl Into<Cow<'a, str>>, ver: impl Into<Cow<'a, str>>) -> Self {
        ModIconPage { id: id.into(), v: Some(ver.into()) }
    }

    pub fn link(&self, req: &HttpRequest) -> Url {
        let mut url = req.url_for_static("mod_icon_page").unwrap();
        url.set_query(Some(&serde_urlencoded::to_string(self).unwrap()));
        url
    }
}
\ No newline at end of file

M srvrc => srvrc +4 -5
@@ 3,12 3,11 @@ listen [::1]:8000

hostname localhost

db_url "postgres://modsite@localhost/modsite"
db_url "postgres://modsite:modsite@localhost/modsite"

// look ma no threads
http_threads 1
db_pool_size 1
http_threads_scale 2
db_pool_size_scale 2
db_pool_size_min 1

source_dir data/
// scan
\ No newline at end of file
// scan

M static/mod.css => static/mod.css +40 -5
@@ 1,12 1,28 @@
#mod-info-bar {
    display: flex;
}

#mod-icon {
    width: 50px;
    height: 50px;
    outline: 1px solid #888;
    width: 64px;
    height: 64px;
    border: 1px solid #888;
    vertical-align: middle;
    flex: none;
    image-rendering: pixelated;
}

@supports (-moz-appearance: none) {
    #mod-icon {
        image-rendering: optimizeSpeed;
    }
}

#mod-text {
    vertical-align: middle;
    flex: none;
    margin: 2px 10px 2px;
    display: flex;
    flex-direction: column;
}

#mod-name {


@@ 20,11 36,12 @@
}

#top-links {
    margin-left: 55px;
    margin-top: auto;
    flex: none;
}

#top-links a {
    margin: 0 5px;
    margin-right: 5px;
}

.deplist .deprange {


@@ 32,4 49,22 @@
    font-style: italic;
    margin-left: 4px;
    margin-right: 4px;
}

#relations {
    display: flex;
}

#dependencies {
    border: 1px solid #888;
    margin: 4px;
    padding: 4px;
    flex: 0 0 50%;
}

#required-by {
    border: 1px solid #888;
    margin: 4px;
    padding: 4px;
    flex: 1;
}
\ No newline at end of file

M static/modlist.css => static/modlist.css +11 -4
@@ 38,15 38,22 @@
}

#modlist tr {
    outline: 1px solid #888;
    border: 1px solid #888;
    padding-bottom: 4px;
    padding-top: 4px;
}

.mod-icon img {
    width: 50px;
    height: 50px;
    outline: 1px solid #888;
    width: 64px;
    height: 64px;
    border: 1px solid #888;
    image-rendering: pixelated;
}

@supports (-moz-appearance: none) {
    .mod-icon img {
        image-rendering: optimizeSpeed;
    }
}

.mod-body {

M templates/index.html.hbs => templates/index.html.hbs +1 -1
@@ 23,7 23,7 @@
        {{#each mods}}
          <tr class="mod">
            <td class="mod-icon">
              <img alt="{{display_name}} icon" src="/static/icon.png">
              <img alt="{{display_name}} icon" src="/mod/icon?id={{from_db/id}}">
            </td>
            <td class="mod-body">
              <div>

M templates/mod.html.hbs => templates/mod.html.hbs +28 -20
@@ 8,14 8,16 @@
  <body>
    <h1 id="page-title">Fabric Mod Links</h1>
    <hr>
    <div>
      <img id="mod-icon" src="/static/icon.png">
      <span id="mod-text"><span id="mod-name">{{name}}</span><span id="mod-version">{{from_db/file_version}}</span></span>
    </div>
    <div id="top-links">
      {{#if from_db/homepage}}<a href="{{from_db/homepage}}">homepage</a>{{/if}}
      {{#if from_db/issues}}<a href="{{from_db/issues}}">issues</a>{{/if}}
      {{#if from_db/source}}<a href="{{from_db/source}}">source</a>{{/if}}
    <div id="mod-info-bar">
      <img id="mod-icon" src="/mod/icon?id={{id}}">
      <div id="mod-text">
        <span id="mod-name-version"><span id="mod-name">{{name}}</span><span id="mod-version">{{from_db/file_version}}</span></span>
        <div id="top-links">
          {{#if from_db/homepage}}<a href="{{from_db/homepage}}">homepage</a>{{/if}}
          {{#if from_db/issues}}<a href="{{from_db/issues}}">issues</a>{{/if}}
          {{#if from_db/source}}<a href="{{from_db/source}}">source</a>{{/if}}
        </div>
      </div>
    </div>
    <hr>
    <h2>Files</h2>


@@ 31,18 33,24 @@
    {{#each allmods}}
      {{>mod_in_vlist}}
    {{/each}}
    <h2>Dependencies ({{dependencies_count}})</h2>
    <ul id="dependencies" class="deplist">
      {{#each dependencies}}
        <li>{{#if link}}<a href="{{link}}">{{mod_name}}</a>{{else}}{{id}}{{/if}}<span class="deprange">{{range}}</span>{{#if optional}} <i>(optional)</i>{{/if}}</li>
      {{/each}}
    </ul>
    <h2>Required by ({{required_by_count}})</h2>
    <ul id="required-by" class="deplist">
      {{#each required_by}}
        <li>{{mod_name}} ({{#each versions}}<a href="{{link}}">{{version}}</a>{{#if @last}}{{else}}, {{/if}}{{/each}}){{#if optional}} <i>(optional)</i>{{/if}}</li>
      {{/each}}
    </ul>
    <div id="relations">
      <div id="dependencies">
        <h2>Dependencies ({{dependencies_count}})</h2>
        <ul class="deplist">
          {{#each dependencies}}
            <li>{{#if link}}<a href="{{link}}">{{mod_name}}</a>{{else}}{{id}}{{/if}}<span class="deprange">{{range}}</span>{{#if optional}} <i>(optional)</i>{{/if}}</li>
          {{/each}}
        </ul>
      </div>
      <div id="required-by">
        <h2>Required by ({{required_by_count}})</h2>
        <ul class="deplist">
          {{#each required_by}}
            <li>{{mod_name}} ({{#each versions}}<a href="{{link}}">{{version}}</a>{{#if @last}}{{else}}, {{/if}}{{/each}}){{#if optional}} <i>(optional)</i>{{/if}}</li>
          {{/each}}
        </ul>
      </div>
    </div>
    <hr>
    {{>footer}}
  </body>