From 7b0eaa806f2cfc84e4c26f8f608e1d4e4843ea05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yaroslav=20de=20la=20Pe=C3=B1a=20Smirnov?= Date: Sun, 28 Nov 2021 04:37:59 +0300 Subject: Clean out old files/dirs Delete extraneous files/images/albums that are no longer present in the source directory. --- README.md | 8 ++- include/components.h | 3 + include/fs.h | 24 ++++++- include/render.h | 3 + include/site.h | 3 +- src/components.c | 2 + src/fs.c | 196 +++++++++++++++++++++++++++++++++++++++++++-------- src/render.c | 10 ++- src/site.c | 59 ++++++++++++---- unja | 2 +- 10 files changed, 256 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index d0f1a8f..da6663f 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,12 @@ A static web image gallery generator. It optimizes images for the web and generates HTML files to create a photo/image gallery web site ready to be served by an HTML server. -VERY EARLY STAGES! It's semifunctional but not very useful yet. +EARLY STAGES! It's functional but still rough around the edges. ## Building +revela depends on GraphicsMagick (1.3+ tested) and libexif (0.6+ tested). + ``` git submodule update --init make @@ -25,8 +27,8 @@ DEBUG=1 make new image was added, the templates for the next and prev images, and the album are not updated accordingly. * Add exif tags to template hashmap. -* Clean out-of-date dirs and files (i.e. removed albums/images from the source - dir). * Improve unja or find better alternative. +* Get rid of previews variable. It is a temporary hack to limit the amount of + thumbnails for each album in the index. * Better test coverage? (if I am not too lazy) * Document. diff --git a/include/components.h b/include/components.h index c56de77..e65a66a 100644 --- a/include/components.h +++ b/include/components.h @@ -57,11 +57,14 @@ struct album { /* The date of the album is the date of the earliest image */ time_t tstamp; struct bstree *images; + /* Files/dirs that belong to images and which shouldn't be deleted */ + struct hashmap *preserved; /* Hashmap with values to be passed to the template */ struct hashmap *map; /* Vector with hashmaps of images to be passed to the templates */ struct vector *thumbs; struct vector *previews; + size_t images_updated; }; struct image *image_new(char *src, const struct stat *, struct album *); diff --git a/include/fs.h b/include/fs.h index 973d585..218958b 100644 --- a/include/fs.h +++ b/include/fs.h @@ -1,10 +1,17 @@ #ifndef REVELA_FS_H #define REVELA_FS_H -#include #include #include +#include "hashmap.h" + +enum nmkdir_res { + NMKDIR_ERROR, + NMKDIR_NOOP, + NMKDIR_CREATED, +}; + /* * Returns a pointer to where the basename of the file is inside path. */ @@ -14,7 +21,7 @@ const char *rbasename(const char *path); * Makes a new directory if it doesn't exist. If there were errors returns * false, otherwise returns true. */ -bool nmkdir(const char *path, struct stat *dstat, bool dry); +enum nmkdir_res nmkdir(const char *path, struct stat *dstat, bool dry); #define joinpathb(buf, a, b) sprintf(buf, "%s/%s", a, b) @@ -42,6 +49,16 @@ int file_is_uptodate(const char *path, const struct timespec *srcmtim); */ void setdatetime(const char *path, const struct timespec *mtim); +bool rmentry(const char *path); + +/* + * Recursively deletes extaneous files from directory, keeping files in the + * preserved hashmap. Returns -1 on error, number of deleted entries on success. + * The number is not the total number of files on all subdirectories, but only + * the number of files/dirs deleted from the directory pointed by path. + */ +ssize_t rmextra(const char *path, struct hashmap *preserved); + /* * Copies file(s) truncating and overwritting the file(s) in the destination * path if they exist and are not a directory, or creating them if they don't @@ -49,6 +66,7 @@ void setdatetime(const char *path, const struct timespec *mtim); * recursively. Only copies the regular files that already exist if their * timestamps do not match. */ -bool filesync(const char *restrict srcpath, const char *restrict dstpath); +bool filesync(const char *restrict srcpath, const char *restrict dstpath, + struct hashmap *preserved); #endif diff --git a/include/render.h b/include/render.h index faa6f73..ff83bab 100644 --- a/include/render.h +++ b/include/render.h @@ -19,6 +19,7 @@ struct render { struct hashmap *common_vars; /* Modification time for the templates dir */ struct timespec modtime; + size_t albums_updated; bool dry_run; }; @@ -30,6 +31,8 @@ bool render_make_album(struct render *r, const char *path, bool render_make_image(struct render *r, const char *path, const struct image *image); +bool render_set_album_vars(struct render *, struct album *); + bool render_init(struct render *, const char *path, struct site_config *, struct bstree *albums); diff --git a/include/site.h b/include/site.h index 678cd47..6ddac82 100644 --- a/include/site.h +++ b/include/site.h @@ -12,7 +12,6 @@ #define SITE_DEFAULT_RESOURCES "/usr/share/revela" #endif -#define CSSDIR "css" #define STATICDIR "static" #define CONTENTDIR "content" #define TEMPLATESDIR "templates" @@ -31,6 +30,8 @@ struct site { */ size_t rel_content_dir; struct bstree *albums; + /* Files/dirs that belong to albums and which shouldn't be deleted */ + struct hashmap *album_dirs; struct render render; bool dry_run; }; diff --git a/src/components.c b/src/components.c index 8937a83..2e3bc5e 100644 --- a/src/components.c +++ b/src/components.c @@ -199,6 +199,7 @@ album_new(struct album_config *conf, struct site_config *sconf, const char *src, album->slug = slugify(rsrc, sconf->base_url, &album->url); album->images = bstree_new(image_cmp, image_destroy); album->tstamp = MAXTIME; + album->image_dirs = hashmap_new(); album->map = hashmap_new_with_cap(16); album->thumbs = vector_new(128); album->previews = vector_new(sconf->max_previews); @@ -241,6 +242,7 @@ album_destroy(void *data) free(album->source); free(album->url); bstree_destroy(album->images); + hashmap_free(album->image_dirs); hashmap_free(album->map); vector_free(album->thumbs); vector_free(album->previews); diff --git a/src/fs.c b/src/fs.c index a1b5626..58bb43d 100644 --- a/src/fs.c +++ b/src/fs.c @@ -2,6 +2,7 @@ #include "log.h" #include "config.h" +#include "vector.h" #include #include @@ -35,6 +36,12 @@ static const char *img_extensions[] = { NULL, }; +static void +hm_destroy_callback(const void *k, void *v) +{ + free(v); +} + const char * rbasename(const char *path) { @@ -45,44 +52,43 @@ rbasename(const char *path) return delim + 1; } -bool +enum nmkdir_res nmkdir(const char *path, struct stat *dstat, bool dry) { if (dry) { if (stat(path, dstat)) { if (errno == ENOENT) { log_printl(LOG_DETAIL, "Created directory %s", path); - return true; + return NMKDIR_CREATED; } log_printl_errno(LOG_FATAL, "Can't read %s", path); - return false; + return NMKDIR_ERROR; } if (!S_ISDIR(dstat->st_mode)) { log_printl(LOG_FATAL, "%s is not a directory", path); - return false; + return NMKDIR_ERROR; } - return true; + return NMKDIR_NOOP; } if(mkdir(path, 0755) < 0) { if (errno == EEXIST) { if (stat(path, dstat)) { log_printl_errno(LOG_FATAL, "Can't read %s", path); - return false; + return NMKDIR_ERROR; } if (!S_ISDIR(dstat->st_mode)) { log_printl(LOG_FATAL, "%s is not a directory", path); - return false; + return NMKDIR_ERROR; } + return NMKDIR_NOOP; } else { log_printl_errno(LOG_FATAL, "Can't make directory %s", path); - return false; + return NMKDIR_ERROR; } - } else { - log_printl(LOG_DETAIL, "Created directory %s", path); } - - return true; + log_printl(LOG_DETAIL, "Created directory %s", path); + return NMKDIR_CREATED; } char * @@ -151,26 +157,126 @@ setdatetime(const char *path, const struct timespec *mtim) } bool -filesync(const char *restrict srcpath, const char *restrict dstpath) +rmentry(const char *path) +{ + struct stat st; + if (stat(path, &st)) { + log_printl_errno(LOG_ERROR, "Can't stat file %s", path); + return false; + } + + if (S_ISDIR(st.st_mode)) { + DIR *dir = opendir(path); + struct dirent *ent; + while ((ent = readdir(dir))) { + if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { + continue; + } + + char target[PATH_MAX]; + sprintf(target, "%s/%s", path, ent->d_name); + if (!rmentry(target)) { + closedir(dir); + return false; + } + } + + closedir(dir); + if (rmdir(path)) goto error; + goto success; + } + + if (unlink(path)) goto error; + +success: + log_printl(LOG_DETAIL, "Deleted %s", path); + return true; +error: + log_printl_errno(LOG_ERROR, "Can't delete %s", path); + return false; +} + +ssize_t +rmextra(const char *path, struct hashmap *preserved) +{ + ssize_t removed = 0; + DIR *dir = opendir(path); + if (dir == NULL) return -1; + + struct dirent *ent; + while ((ent = readdir(dir))) { + if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { + continue; + } + + if (hashmap_get(preserved, ent->d_name) != NULL) continue; + + char target[PATH_MAX]; + sprintf(target, "%s/%s", path, ent->d_name); + if (!rmentry(target)) { + closedir(dir); + return -1; + } + removed++; + } + + closedir(dir); + return removed; +} + +bool +filesync(const char *restrict srcpath, const char *restrict dstpath, + struct hashmap *preserved) { int fdsrc; struct stat stsrc; + struct vector *own = NULL; + bool cleanup = false; fdsrc = open(srcpath, O_RDONLY); - if (fdsrc < 0) return false; - if (fstat(fdsrc, &stsrc)) goto dir_error; + if (fdsrc < 0) { + log_printl_errno(LOG_ERROR, "Failed to open %s", srcpath); + return false; + } + if (fstat(fdsrc, &stsrc)) { + log_printl_errno(LOG_ERROR, "Failed to stat %s", srcpath); + goto dir_error; + } if (S_ISDIR(stsrc.st_mode)) { if (mkdir(dstpath, 0755)) { - if (errno != EEXIST) goto dir_error; - if (stat(dstpath, &stsrc)) goto dir_error; + if (errno != EEXIST) { + log_printl_errno(LOG_ERROR, "Failed to create directory %s", + dstpath); + goto dir_error; + } + if (stat(dstpath, &stsrc)) { + log_printl_errno(LOG_ERROR, "Failed to stat %s", dstpath); + goto dir_error; + } if (!S_ISDIR(stsrc.st_mode)){ + log_printl(LOG_ERROR, "%s is not a directory", dstpath); errno = ENOTDIR; goto dir_error; } + /* We only need to cleanup if the dir already existed */ + cleanup = true; + } + + if (cleanup) { + if (preserved == NULL) { + preserved = hashmap_new(); + } else { + own = vector_new(32); + } } + DIR *dir = fdopendir(fdsrc); - if (dir == NULL) goto dir_error; + if (dir == NULL) { + log_printl_errno(LOG_ERROR, "Failed to open directory %s", dstpath); + goto dir_error; + } + struct dirent *ent; while ((ent = readdir(dir))) { if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { @@ -179,37 +285,71 @@ filesync(const char *restrict srcpath, const char *restrict dstpath) char entsrc[PATH_MAX], entdst[PATH_MAX]; sprintf(entsrc, "%s/%s", srcpath, ent->d_name); sprintf(entdst, "%s/%s", dstpath, ent->d_name); - if (filesync(entsrc, entdst)) { + if (cleanup) { + char *name = strdup(ent->d_name); + hashmap_insert(preserved, name, name); + if (own) { + vector_push(own, name); + } + } + if (!filesync(entsrc, entdst, NULL)) { closedir(dir); - goto dir_error; + return false; } } - closedir(dir); - goto dir_success; + if (cleanup) { + rmextra(dstpath, preserved); + if (own) { + for (size_t i = 0; i < own->size; i++) { + free(own->values[i]); + } + vector_free(own); + } else { + hashmap_destroy(preserved, hm_destroy_callback); + } + } + + closedir(dir); + return true; } int fddst, uptodate; if ((uptodate = file_is_uptodate(dstpath, &stsrc.st_mtim)) > 0) { - goto dir_success; + goto success; } else if (uptodate < 0) { goto dir_error; } fddst = open(dstpath, O_CREAT | O_WRONLY | O_TRUNC, 0644); - if (fddst < 0) goto dir_error; + if (fddst < 0) { + log_printl_errno(LOG_ERROR, "Failed to open/create %s", dstpath); + goto dir_error; + } #ifdef __linux__ ssize_t nwrote = sendfile(fddst, fdsrc, NULL, stsrc.st_size); - if (nwrote != stsrc.st_size) goto copy_error; + if (nwrote != stsrc.st_size) { + log_printl_errno(LOG_ERROR, "Failed to copy %s (wrote %lu/%lu bytes)", + dstpath, nwrote, stsrc.st_size); + goto copy_error; + } #else char buf[BUFSIZE]; ssize_t nread; while ((nread = read(fdsrc, buf, BUFSIZE)) > 0) { ssize_t nwrote = write(fddst, buf, nread); - if (nread != nwrote) goto copy_error; + if (nread != nwrote) { + log_printl_errno(LOG_ERROR, "Failed to copy %s " + "(buffer wrote %lu/%lu bytes)", + dstpath, nwrote, nread); + goto copy_error; + } + } + if (nread < 0) { + log_printl_errno(LOG_ERROR, "Failed to copy %s"); + goto copy_error; } - if (nread < 0) goto copy_error; #endif struct timespec tms[] = { @@ -221,7 +361,7 @@ filesync(const char *restrict srcpath, const char *restrict dstpath) log_printl(LOG_DETAIL, "Copied %s", srcpath); close(fddst); -dir_success: +success: close(fdsrc); return true; copy_error: diff --git a/src/render.c b/src/render.c index acd1de2..79fb543 100644 --- a/src/render.c +++ b/src/render.c @@ -84,11 +84,9 @@ years_destroy(struct vector *years) vector_free(years); } -static bool -albums_walk(struct bstnode *node, void *data) +bool +render_set_album_vars(struct render *r, struct album *album) { - struct album *album = node->value; - struct render *r = data; hashmap_insert(album->map, "title", album->config->title); hashmap_insert(album->map, "desc", album->config->desc); @@ -213,7 +211,7 @@ render_init(struct render *r, const char *root, struct site_config *conf, r->years = vector_new(8); r->albums = vector_new(64); - r->common_vars = hashmap_new_with_cap(8); + r->common_vars = hashmap_new_with_cap(16); hashmap_insert(r->common_vars, "title", conf->title); if (strlen(conf->base_url) == 0) { hashmap_insert(r->common_vars, "index", "/"); @@ -221,7 +219,7 @@ render_init(struct render *r, const char *root, struct site_config *conf, hashmap_insert(r->common_vars, "index", conf->base_url); } - bstree_inorder_walk(albums->root, albums_walk, (void *)r); + //bstree_inorder_walk(albums->root, albums_walk, (void *)r); cleanup: free(tmplpath); diff --git a/src/site.c b/src/site.c index 5fa40c4..6fe8d85 100644 --- a/src/site.c +++ b/src/site.c @@ -9,12 +9,15 @@ #include "fs.h" #include "log.h" #include "hashmap.h" +#include "render.h" /* TODO: Probably shouldn't use PATH_MAX, but i'll leave it for now */ /* TODO: handle error cases for paths that are too long */ #define THUMB_SUFFIX "_thumb" +static const char *index_html = "index.html"; + static bool wand_passfail(MagickWand *wand, MagickPassFail status) { @@ -80,6 +83,8 @@ images_walk(struct bstnode *node, void *data) struct site *site = data; struct image *image = node->value; struct stat dstat; + char htmlpath[PATH_MAX]; + const char *base = rbasename(image->dst); log_printl(LOG_DEBUG, "Image: %s, datetime %s", image->basename, image->datestr); @@ -101,8 +106,8 @@ images_walk(struct bstnode *node, void *data) MagickRemoveImage(site->wand); } - char htmlpath[PATH_MAX]; - joinpathb(htmlpath, image->dst, "index.html"); + joinpathb(htmlpath, image->dst, index_html); + hashmap_insert(image->album->preserved, base, (char *)base); return render_make_image(&site->render, htmlpath, image); magick_fail: return false; @@ -116,12 +121,27 @@ albums_walk(struct bstnode *node, void *data) struct stat dstat; if (!nmkdir(album->slug, &dstat, site->dry_run)) return false; + if (!site->dry_run) { + hashmap_insert(site->album_dirs, album->slug, (char *)album->slug); + if (!render_set_album_vars(&site->render, album)) return false; + } + char htmlpath[PATH_MAX]; - joinpathb(htmlpath, album->slug, "index.html"); + joinpathb(htmlpath, album->slug, index_html); if (!render_make_album(&site->render, htmlpath, album)) return false; log_printl(LOG_DEBUG, "Album: %s, datetime %s", album->slug, album->datestr); - return bstree_inorder_walk(album->images->root, images_walk, site); + if (!bstree_inorder_walk(album->images->root, images_walk, site)) { + return false; + } + + hashmap_insert(album->preserved, index_html, (char *)index_html); + if (rmextra(album->slug, album->preserved) < 0) { + log_printl_errno(LOG_ERROR, + "Something happened while deleting extraneous files"); + } + + return true; } /* @@ -201,24 +221,35 @@ site_build(struct site *site) if (!nmkdir(site->output_dir, &dstat, false)) return false; - joinpathb(staticp, site->root_dir, STATICDIR); - if (!filesync(staticp, site->output_dir) && errno != ENOENT) { - log_printl_errno(LOG_FATAL, "Can't copy static files"); - return false; - } - if (chdir(site->output_dir)) { log_printl_errno(LOG_FATAL, "Can't change to directory %s", site->output_dir); return false; } - if (!render_make_index(&site->render, "index.html")) return false; - if (!bstree_inorder_walk(site->albums->root, albums_walk, (void *)site)) { return false; } + if (!render_make_index(&site->render, index_html)) return false; + hashmap_insert(site->album_dirs, index_html, (char *)index_html); + + joinpathb(staticp, site->root_dir, STATICDIR); + if (stat(staticp, &dstat)) { + if (errno != ENOENT) { + log_printl_errno(LOG_FATAL, "Couldn't read static dir"); + return false; + } + if (rmextra(site->output_dir, site->album_dirs) < 0) { + log_printl_errno(LOG_ERROR, + "Something happened while deleting extraneous files"); + } + } else if (!site->dry_run + && !filesync(staticp, site->output_dir, site->album_dirs)) { + log_printl(LOG_FATAL, "Can't copy static files"); + return false; + } + chdir(site->root_dir); return true; } @@ -256,6 +287,9 @@ site_init(struct site *site) site->rel_content_dir = strlen(site->root_dir) + 1; InitializeMagick(NULL); site->wand = NewMagickWand(); + if (!site->dry_run) { + site->album_dirs = hashmap_new(); + } site->render.dry_run = site->dry_run; return true; @@ -273,6 +307,7 @@ site_deinit(struct site *site) DestroyMagick(); } if (!site->dry_run) { + hashmap_free(site->album_dirs); render_deinit(&site->render); } } diff --git a/unja b/unja index 8a54056..bf330c1 160000 --- a/unja +++ b/unja @@ -1 +1 @@ -Subproject commit 8a5405629f7dcbc2504ac55f57775180a011b846 +Subproject commit bf330c19cd3813da8a9c0ae4f48316747f3fa93a -- cgit v1.2.3