vertigo-cli 0.14.5

Reactive Real-DOM library with SSR for Rust - packaging/serving tool
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
use actix_files::NamedFile;
use actix_web::{
    HttpRequest, HttpResponse, Resource, guard,
    http::header::{
        self, AcceptEncoding, ETag, Encoding, EntityTag, Header, HeaderValue, HttpDate,
        IfNoneMatch, LastModified,
    },
    web::{self, Bytes},
};
use std::{
    collections::HashMap,
    ffi::OsString,
    io::{Read, Write},
    path::{Path, PathBuf},
    sync::Arc,
    time::Instant,
};
use walkdir::{DirEntry, WalkDir};

use crate::build::get_hash;

/// The build's files compressed by [`precompress`], by the path they are served at
pub type Precompressed = HashMap<String, PrecompressedFile>;

pub struct PrecompressedFile {
    /// The file as it is, for the clients not accepting brotli
    path: PathBuf,
    content_type: String,
    brotli: Bytes,
    etag: EntityTag,
    last_modified: Option<HttpDate>,
}

/// Formats worth it: images and fonts come compressed already
const COMPRESSIBLE: &[&str] = &[
    "wasm", "js", "mjs", "map", "css", "json", "svg", "html", "txt", "xml",
];

/// Compresses the files in `dest_dir` with brotli at its highest level, too slow to run per
/// response, to serve them under `http_root`. A file `vertigo build` saved compressed next to
/// it ([`save_precompressed`]) is taken as it is, when it unpacks to the file.
pub fn precompress(dest_dir: impl AsRef<Path>, http_root: &str) -> Precompressed {
    let dest_dir = dest_dir.as_ref();

    // The server answers nothing until it's done, so say what it waits for
    if cfg!(debug_assertions) {
        log::warn!(
            "Precompressing {} with brotli in a debug build, unoptimized: a debug wasm takes \
             minutes unless vertigo build saved it compressed. \
             MountConfigBuilder::precompress_in_release skips it here",
            dest_dir.display()
        );
    } else {
        log::info!("Precompressing {} with brotli...", dest_dir.display());
    }

    let started = Instant::now();
    let mut files = Precompressed::new();
    let (mut size, mut compressed_size, mut from_build) = (0, 0, 0);

    for (entry, extension) in compressible_files(dest_dir) {
        let path = entry.path();

        let (content, brotli, saved) = match read_compressed(path) {
            Ok(compressed) => compressed,
            Err(err) => {
                log::warn!("Can't precompress {}: {err}", path.display());
                continue;
            }
        };

        // A small file can come out bigger
        if brotli.len() >= content.len() {
            continue;
        }

        let Ok(relative) = path.strip_prefix(dest_dir) else {
            continue;
        };

        let url_path = relative
            .components()
            .map(|component| component.as_os_str().to_string_lossy())
            .collect::<Vec<_>>()
            .join("/");

        size += content.len();
        compressed_size += brotli.len();
        from_build += usize::from(saved);

        files.insert(
            format!("{}/{url_path}", http_root.trim_end_matches('/')),
            PrecompressedFile {
                path: path.to_path_buf(),
                content_type: content_type(extension),
                etag: EntityTag::new_strong(get_hash(&brotli)),
                last_modified: entry
                    .metadata()
                    .ok()
                    .and_then(|metadata| metadata.modified().ok())
                    .map(HttpDate::from),
                brotli: Bytes::from(brotli),
            },
        );
    }

    log::info!(
        "Precompressed {} files with brotli ({from_build} saved by the build): {}K -> {}K in {:.1?}",
        files.len(),
        size / 1024,
        compressed_size / 1024,
        started.elapsed()
    );

    files
}

/// Saves the text files in `dest_dir` compressed with brotli next to them, `name.ext.br`, for
/// [`precompress`] to take at the server's start instead of compressing them then. A file
/// brotli makes bigger gets none. Gives how many it saved, their size and compressed size.
pub(crate) fn save_precompressed(dest_dir: &Path) -> std::io::Result<(usize, usize, usize)> {
    let (mut count, mut size, mut compressed_size) = (0, 0, 0);

    for (entry, _) in compressible_files(dest_dir) {
        let path = entry.path();
        let content = std::fs::read(path)?;
        let brotli = brotli_compress(&content)?;

        if brotli.len() >= content.len() {
            continue;
        }

        std::fs::write(saved_path(path), &brotli)?;
        count += 1;
        size += content.len();
        compressed_size += brotli.len();
    }

    Ok((count, size, compressed_size))
}

/// The files worth compressing - text ones, not hidden, as `Files` doesn't serve hidden ones -
/// with their extension
fn compressible_files(dest_dir: &Path) -> impl Iterator<Item = (DirEntry, &'static str)> {
    WalkDir::new(dest_dir)
        .into_iter()
        .filter_entry(|entry| entry.depth() == 0 || !is_hidden(entry))
        .filter_map(Result::ok)
        .filter(|entry| entry.file_type().is_file())
        .filter_map(|entry| {
            let extension = entry.path().extension()?.to_str()?;
            let extension = COMPRESSIBLE.iter().find(|known| **known == extension)?;
            Some((entry, *extension))
        })
}

fn is_hidden(entry: &DirEntry) -> bool {
    entry.file_name().to_string_lossy().starts_with('.')
}

/// Where [`save_precompressed`] puts the file compressed: `name.ext.br`
fn saved_path(path: &Path) -> PathBuf {
    let mut name = OsString::from(path.as_os_str());
    name.push(".br");
    PathBuf::from(name)
}

/// The file, the file compressed, and whether it's the one the build saved. That one counts
/// only when it unpacks to the file: a build copied without its times, or a file changed after
/// the build, gets compressed again.
fn read_compressed(path: &Path) -> std::io::Result<(Vec<u8>, Vec<u8>, bool)> {
    let content = std::fs::read(path)?;

    if let Ok(saved) = std::fs::read(saved_path(path))
        && unpacks_to(&saved, &content)
    {
        return Ok((content, saved, true));
    }

    let brotli = brotli_compress(&content)?;
    Ok((content, brotli, false))
}

fn unpacks_to(brotli: &[u8], content: &[u8]) -> bool {
    let mut unpacked = Vec::with_capacity(content.len());
    brotli::Decompressor::new(brotli, 64 * 1024)
        .read_to_end(&mut unpacked)
        .is_ok_and(|_| unpacked == content)
}

/// Compresses at quality 11, where `Compress` uses 3 to keep up with responses
pub(crate) fn brotli_compress(content: &[u8]) -> std::io::Result<Vec<u8>> {
    let mut writer = brotli::CompressorWriter::new(Vec::new(), 64 * 1024, 11, 22);
    writer.write_all(content)?;
    Ok(writer.into_inner())
}

/// What `Files` sends, which adds the charset to text (`prefer_utf8`)
fn content_type(extension: &str) -> String {
    let mime = actix_files::file_extension_to_mime(extension);

    match mime.essence_str() {
        "application/javascript"
        | "text/html"
        | "text/css"
        | "text/plain"
        | "text/csv"
        | "text/tab-separated-values" => format!("{mime}; charset=utf-8"),
        _ => mime.to_string(),
    }
}

/// Serves the files [`precompress`] compressed: to the browsers accepting brotli compressed, to
/// the other clients as they are. Other paths fall through to the next service.
pub fn precompressed_files(http_root: &str, files: Arc<Precompressed>) -> Resource {
    let pattern = format!("{}/{{file:.*}}", http_root.trim_end_matches('/'));
    let served = files.clone();

    web::resource(pattern)
        .guard(guard::Any(guard::Get()).or(guard::Head()))
        .guard(guard::fn_guard(move |ctx| {
            served.contains_key(ctx.head().uri.path())
        }))
        .to(move |req: HttpRequest| {
            let files = files.clone();
            async move { respond(&req, files.get(req.path())) }
        })
}

fn accepts_brotli(req: &HttpRequest) -> bool {
    // A range is of the file as it is
    if req.headers().contains_key(header::RANGE) {
        return false;
    }

    AcceptEncoding::parse(req)
        .ok()
        .and_then(|accept| accept.negotiate([Encoding::brotli()].iter()))
        == Some(Encoding::brotli())
}

/// Both versions say they depend on `Accept-Encoding`, or a cache on the way (a proxy, a CDN)
/// would give everyone the one it got first - f. ex. the file as it is, which a crawler without
/// `Accept-Encoding` asked for.
fn respond(req: &HttpRequest, file: Option<&PrecompressedFile>) -> actix_web::Result<HttpResponse> {
    let Some(file) = file else {
        return Ok(HttpResponse::NotFound().finish());
    };

    let mut response = if accepts_brotli(req) {
        brotli_response(req, file)
    } else {
        // What `Files` would send, it has the same defaults
        NamedFile::open(&file.path)?.into_response(req)
    };

    // `Compress`, compressing the file as it is for a client accepting only gzip, appends one
    // more, the same
    response
        .headers_mut()
        .append(header::VARY, HeaderValue::from_static("accept-encoding"));

    Ok(response)
}

fn brotli_response(req: &HttpRequest, file: &PrecompressedFile) -> HttpResponse {
    let unchanged = match IfNoneMatch::parse(req) {
        Ok(IfNoneMatch::Any) => true,
        Ok(IfNoneMatch::Items(etags)) => etags.iter().any(|etag| etag.weak_eq(&file.etag)),
        Err(_) => false,
    };

    let mut response = if unchanged {
        HttpResponse::NotModified()
    } else {
        HttpResponse::Ok()
    };

    response.insert_header(ETag(file.etag.clone()));

    if let Some(last_modified) = file.last_modified {
        response.insert_header(LastModified(last_modified));
    }

    if unchanged {
        return response.finish();
    }

    // `Compress` leaves a response that has its `Content-Encoding` alone
    response
        .insert_header((header::CONTENT_TYPE, file.content_type.as_str()))
        .insert_header((header::CONTENT_ENCODING, "br"))
        .body(file.brotli.clone())
}

#[cfg(test)]
mod tests {
    use actix_files::Files;
    use actix_web::{
        App,
        dev::ServiceResponse,
        http::{
            Method, StatusCode,
            header::{self, HeaderName},
        },
        test::{TestRequest, call_service, init_service, read_body},
    };
    use std::{
        io::{Read, Write},
        sync::Arc,
    };

    use super::{precompress, precompressed_files, save_precompressed};

    fn header(response: &ServiceResponse, name: HeaderName) -> Option<&str> {
        response
            .headers()
            .get(name)
            .and_then(|value| value.to_str().ok())
    }

    #[actix_web::test]
    async fn brotli_goes_to_browsers_accepting_it() -> std::io::Result<()> {
        let dest_dir =
            std::env::temp_dir().join(format!("vertigo-precompress-{}", std::process::id()));
        std::fs::create_dir_all(dest_dir.join("assets"))?;

        let script = "console.log('vertigo');\n".repeat(100);
        std::fs::write(dest_dir.join("assets/app.js"), &script)?;
        std::fs::write(dest_dir.join("logo.png"), vec![0; 1000])?;
        std::fs::write(dest_dir.join("index.json"), "{}")?;
        std::fs::write(dest_dir.join(".hidden.js"), &script)?;

        let files = precompress(&dest_dir, "/build");
        let mut paths = files.keys().collect::<Vec<_>>();
        paths.sort();
        // Not an image, nor a file brotli makes bigger, nor a hidden one
        assert_eq!(paths, ["/build/assets/app.js"]);

        let app = init_service(
            App::new()
                .service(precompressed_files("/build", Arc::new(files)))
                .service(Files::new("/build", &dest_dir)),
        )
        .await;

        let plain = || TestRequest::get().uri("/build/assets/app.js");
        let get = |accept_encoding: &str| {
            plain().insert_header((header::ACCEPT_ENCODING, accept_encoding.to_string()))
        };

        let response = call_service(&app, get("gzip, deflate, br, zstd").to_request()).await;
        assert_eq!(response.status(), StatusCode::OK);
        assert_eq!(header(&response, header::CONTENT_ENCODING), Some("br"));
        assert_eq!(header(&response, header::VARY), Some("accept-encoding"));
        let Some(etag) = header(&response, header::ETAG).map(str::to_string) else {
            panic!("a precompressed file should carry an ETag");
        };
        let mut decompressed = String::new();
        brotli::Decompressor::new(&read_body(response).await[..], 4096)
            .read_to_string(&mut decompressed)?;
        assert_eq!(decompressed, script);

        let revalidation = get("br")
            .insert_header((header::IF_NONE_MATCH, etag.clone()))
            .to_request();
        let response = call_service(&app, revalidation).await;
        assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
        assert_eq!(header(&response, header::VARY), Some("accept-encoding"));
        assert!(read_body(response).await.is_empty());

        let head = TestRequest::default()
            .method(Method::HEAD)
            .uri("/build/assets/app.js")
            .insert_header((header::ACCEPT_ENCODING, "br"))
            .to_request();
        let response = call_service(&app, head).await;
        assert_eq!(response.status(), StatusCode::OK);
        assert_eq!(header(&response, header::CONTENT_ENCODING), Some("br"));
        assert_eq!(header(&response, header::VARY), Some("accept-encoding"));

        // The file as it is, also saying it depends on `Accept-Encoding`: a cache that got it
        // first, f. ex. for a crawler without the header, must not give it to the browsers
        let as_it_is = [get("gzip"), get("br;q=0, gzip"), get("identity"), plain()];
        let mut raw_etag = None;
        for request in as_it_is {
            let response = call_service(&app, request.to_request()).await;
            assert_eq!(response.status(), StatusCode::OK);
            assert_eq!(header(&response, header::CONTENT_ENCODING), None);
            assert_eq!(header(&response, header::VARY), Some("accept-encoding"));
            raw_etag = header(&response, header::ETAG).map(str::to_string);
            assert_eq!(read_body(response).await, script.as_bytes());
        }
        let Some(raw_etag) = raw_etag else {
            panic!("the file as it is should carry an ETag");
        };
        assert_ne!(raw_etag, etag);

        let revalidation = plain()
            .insert_header((header::IF_NONE_MATCH, raw_etag))
            .to_request();
        let response = call_service(&app, revalidation).await;
        assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
        assert_eq!(header(&response, header::VARY), Some("accept-encoding"));

        let range = get("br")
            .insert_header((header::RANGE, "bytes=0-9"))
            .to_request();
        let response = call_service(&app, range).await;
        assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
        assert_eq!(header(&response, header::VARY), Some("accept-encoding"));
        assert_eq!(read_body(response).await, script.as_bytes()[..10]);

        // Others than the precompressed ones are left to `Files`
        let response =
            call_service(&app, TestRequest::get().uri("/build/logo.png").to_request()).await;
        assert_eq!(response.status(), StatusCode::OK);
        assert_eq!(header(&response, header::VARY), None);

        std::fs::remove_dir_all(&dest_dir)
    }

    fn unpack(brotli: &[u8]) -> std::io::Result<String> {
        let mut unpacked = String::new();
        brotli::Decompressor::new(brotli, 4096).read_to_string(&mut unpacked)?;
        Ok(unpacked)
    }

    #[test]
    fn the_server_takes_what_the_build_saved_precompressed() -> std::io::Result<()> {
        let dest_dir =
            std::env::temp_dir().join(format!("vertigo-saved-brotli-{}", std::process::id()));
        std::fs::create_dir_all(&dest_dir)?;
        let script = "console.log('vertigo');\n".repeat(100);
        std::fs::write(dest_dir.join("app.js"), &script)?;
        std::fs::write(dest_dir.join("logo.png"), vec![0; 1000])?;
        std::fs::write(dest_dir.join("index.json"), "{}")?;

        let (count, size, _) = save_precompressed(&dest_dir)?;
        assert_eq!((count, size), (1, script.len()));
        // Not an image, nor a file brotli makes bigger
        assert!(dest_dir.join("app.js.br").exists());
        assert!(!dest_dir.join("logo.png.br").exists());
        assert!(!dest_dir.join("index.json.br").exists());
        assert_eq!(unpack(&std::fs::read(dest_dir.join("app.js.br"))?)?, script);

        // Taken as it is: compressed by something else than the server would, it stays so
        let mut quick = brotli::CompressorWriter::new(Vec::new(), 4096, 1, 22);
        quick.write_all(script.as_bytes())?;
        let quick = quick.into_inner();
        std::fs::write(dest_dir.join("app.js.br"), &quick)?;
        let files = precompress(&dest_dir, "/build");
        assert_eq!(files.keys().collect::<Vec<_>>(), ["/build/app.js"]);
        assert_eq!(files["/build/app.js"].brotli, quick);

        // Of a file changed after the build: compressed again
        let changed = "console.log('changed');\n".repeat(100);
        std::fs::write(dest_dir.join("app.js"), &changed)?;
        let files = precompress(&dest_dir, "/build");
        assert_eq!(unpack(&files["/build/app.js"].brotli)?, changed);

        std::fs::remove_dir_all(&dest_dir)
    }
}