Clean out old files/dirs

Delete extraneous files/images/albums that are no longer present in the
source directory.

10 files changed, 255 insertions, 53 deletions

diff --git a/README.md b/README.md
index d0f1a8f..da6663f 100644
--- a/README.md
+++ b/README.md
@@ -4,10 +4,12 @@ A static web image gallery generator. It optimizes images for the web and
 generates HTML files to create a photo/image gallery web site ready to be served
 by an HTML server.
 
-VERY EARLY STAGES! It's semifunctional but not very useful yet.
+EARLY STAGES! It's functional but still rough around the edges.
 
 ## Building
 
+revela depends on GraphicsMagick (1.3+ tested) and libexif (0.6+ tested).
+
 ```
 git submodule update --init
 make
@@ -25,8 +27,8 @@ DEBUG=1 make
   new image was added, the templates for the next and prev images, and the album
   are not updated accordingly.
 * Add exif tags to template hashmap.
-* Clean out-of-date dirs and files (i.e. removed albums/images from the source
-  dir).
 * Improve unja or find better alternative.
+* Get rid of previews variable. It is a temporary hack to limit the amount of
+  thumbnails for each album in the index.
 * Better test coverage? (if I am not too lazy)
 * Document.
diff --git a/include/components.h b/include/components.h
index c56de77..e65a66a 100644
--- a/include/components.h
+++ b/include/components.h
@@ -57,11 +57,14 @@ struct album {
 	/* The date of the album is the date of the earliest image */
 	time_t tstamp;
 	struct bstree *images;
+	/* Files/dirs that belong to images and which shouldn't be deleted */
+	struct hashmap *preserved;
 	/* Hashmap with values to be passed to the template */
 	struct hashmap *map;
 	/* Vector with hashmaps of images to be passed to the templates */
 	struct vector *thumbs;
 	struct vector *previews;
+	size_t images_updated;
 };
 
 struct image *image_new(char *src, const struct stat *, struct album *);
diff --git a/include/fs.h b/include/fs.h
index 973d585..218958b 100644
--- a/include/fs.h
+++ b/include/fs.h
@@ -1,10 +1,17 @@
 #ifndef REVELA_FS_H
 #define REVELA_FS_H
 
-#include <stdio.h>
 #include <stdbool.h>
 #include <sys/stat.h>
 
+#include "hashmap.h"
+
+enum nmkdir_res {
+	NMKDIR_ERROR,
+	NMKDIR_NOOP,
+	NMKDIR_CREATED,
+};
+
 /* 
  * Returns a pointer to where the basename of the file is inside path.
  */
@@ -14,7 +21,7 @@ const char *rbasename(const char *path);
  * Makes a new directory if it doesn't exist. If there were errors returns
  * false, otherwise returns true.
  */
-bool nmkdir(const char *path, struct stat *dstat, bool dry);
+enum nmkdir_res nmkdir(const char *path, struct stat *dstat, bool dry);
 
 #define joinpathb(buf, a, b) sprintf(buf, "%s/%s", a, b)
 
@@ -42,6 +49,16 @@ int file_is_uptodate(const char *path, const struct timespec *srcmtim);
  */
 void setdatetime(const char *path, const struct timespec *mtim);
 
+bool rmentry(const char *path);
+
+/* 
+ * Recursively deletes extaneous files from directory, keeping files in the
+ * preserved hashmap. Returns -1 on error, number of deleted entries on success.
+ * The number is not the total number of files on all subdirectories, but only
+ * the number of files/dirs deleted from the directory pointed by path.
+ */
+ssize_t rmextra(const char *path, struct hashmap *preserved);
+
 /* 
  * Copies file(s) truncating and overwritting the file(s) in the destination
  * path if they exist and are not a directory, or creating them if they don't
@@ -49,6 +66,7 @@ void setdatetime(const char *path, const struct timespec *mtim);
  * recursively. Only copies the regular files that already exist if their
  * timestamps do not match.
  */
-bool filesync(const char *restrict srcpath, const char *restrict dstpath);
+bool filesync(const char *restrict srcpath, const char *restrict dstpath,
+		struct hashmap *preserved);
 
 #endif
diff --git a/include/render.h b/include/render.h
index faa6f73..ff83bab 100644
--- a/include/render.h
+++ b/include/render.h
@@ -19,6 +19,7 @@ struct render {
 	struct hashmap *common_vars;
 	/* Modification time for the templates dir */
 	struct timespec modtime;
+	size_t albums_updated;
 	bool dry_run;
 };
 
@@ -30,6 +31,8 @@ bool render_make_album(struct render *r, const char *path,
 bool render_make_image(struct render *r, const char *path,
 		const struct image *image);
 
+bool render_set_album_vars(struct render *, struct album *);
+
 bool render_init(struct render *, const char *path, struct site_config *,
 		struct bstree *albums);
 
diff --git a/include/site.h b/include/site.h
index 678cd47..6ddac82 100644
--- a/include/site.h
+++ b/include/site.h
@@ -12,7 +12,6 @@
 #define SITE_DEFAULT_RESOURCES "/usr/share/revela"
 #endif
 
-#define CSSDIR "css"
 #define STATICDIR "static"
 #define CONTENTDIR "content"
 #define TEMPLATESDIR "templates"
@@ -31,6 +30,8 @@ struct site {
 	 */
 	size_t rel_content_dir;
 	struct bstree *albums;
+	/* Files/dirs that belong to albums and which shouldn't be deleted */
+	struct hashmap *album_dirs;
 	struct render render;
 	bool dry_run;
 };
diff --git a/src/components.c b/src/components.c
index 8937a83..2e3bc5e 100644
--- a/src/components.c
+++ b/src/components.c
@@ -199,6 +199,7 @@ album_new(struct album_config *conf, struct site_config *sconf, const char *src,
 	album->slug = slugify(rsrc, sconf->base_url, &album->url);
 	album->images = bstree_new(image_cmp, image_destroy);
 	album->tstamp = MAXTIME;
+	album->image_dirs = hashmap_new();
 	album->map = hashmap_new_with_cap(16);
 	album->thumbs = vector_new(128);
 	album->previews = vector_new(sconf->max_previews);
@@ -241,6 +242,7 @@ album_destroy(void *data)
 	free(album->source);
 	free(album->url);
 	bstree_destroy(album->images);
+	hashmap_free(album->image_dirs);
 	hashmap_free(album->map);
 	vector_free(album->thumbs);
 	vector_free(album->previews);
diff --git a/src/fs.c b/src/fs.c
index a1b5626..58bb43d 100644
--- a/src/fs.c
+++ b/src/fs.c
@@ -2,6 +2,7 @@
 
 #include "log.h"
 #include "config.h"
+#include "vector.h"
 
 #include <fcntl.h>
 #include <stdio.h>
@@ -35,6 +36,12 @@ static const char *img_extensions[] = {
 	NULL,
 };
 
+static void
+hm_destroy_callback(const void *k, void *v)
+{
+	free(v);
+}
+
 const char *
 rbasename(const char *path)
 {
@@ -45,44 +52,43 @@ rbasename(const char *path)
 	return delim + 1;
 }
 
-bool
+enum nmkdir_res
 nmkdir(const char *path, struct stat *dstat, bool dry)
 {
 	if (dry) {
 		if (stat(path, dstat)) {
 			if (errno == ENOENT) {
 				log_printl(LOG_DETAIL, "Created directory %s", path);
-				return true;
+				return NMKDIR_CREATED;
 			}
 			log_printl_errno(LOG_FATAL, "Can't read %s", path);
-			return false;
+			return NMKDIR_ERROR;
 		}
 		if (!S_ISDIR(dstat->st_mode)) {
 			log_printl(LOG_FATAL, "%s is not a directory", path);
-			return false;
+			return NMKDIR_ERROR;
 		}
-		return true;
+		return NMKDIR_NOOP;
 	}
 
 	if(mkdir(path, 0755) < 0) {
 		if (errno == EEXIST) {
 			if (stat(path, dstat)) {
 				log_printl_errno(LOG_FATAL, "Can't read %s", path);
-				return false;
+				return NMKDIR_ERROR;
 			}
 			if (!S_ISDIR(dstat->st_mode)) {
 				log_printl(LOG_FATAL, "%s is not a directory", path);
-				return false;
+				return NMKDIR_ERROR;
 			}
+			return NMKDIR_NOOP;
 		} else {
 			log_printl_errno(LOG_FATAL, "Can't make directory %s", path);
-			return false;
+			return NMKDIR_ERROR;
 		}
-	} else {
-		log_printl(LOG_DETAIL, "Created directory %s", path);
 	}
-
-	return true;
+	log_printl(LOG_DETAIL, "Created directory %s", path);
+	return NMKDIR_CREATED;
 }
 
 char *
@@ -151,26 +157,126 @@ setdatetime(const char *path, const struct timespec *mtim)
 }
 
 bool
-filesync(const char *restrict srcpath, const char *restrict dstpath)
+rmentry(const char *path)
+{
+	struct stat st;
+	if (stat(path, &st)) {
+		log_printl_errno(LOG_ERROR, "Can't stat file %s", path);
+		return false;
+	}
+
+	if (S_ISDIR(st.st_mode)) {
+		DIR *dir = opendir(path);
+		struct dirent *ent;
+		while ((ent = readdir(dir))) {
+			if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
+				continue;
+			}
+
+			char target[PATH_MAX];
+			sprintf(target, "%s/%s", path, ent->d_name);
+			if (!rmentry(target)) {
+				closedir(dir);
+				return false;
+			}
+		}
+
+		closedir(dir);
+		if (rmdir(path)) goto error;
+		goto success;
+	}
+
+	if (unlink(path)) goto error;
+
+success:
+	log_printl(LOG_DETAIL, "Deleted %s", path);
+	return true;
+error:
+	log_printl_errno(LOG_ERROR, "Can't delete %s", path);
+	return false;
+}
+
+ssize_t
+rmextra(const char *path, struct hashmap *preserved)
+{
+	ssize_t removed = 0;
+	DIR *dir = opendir(path);
+	if (dir == NULL) return -1;
+
+	struct dirent *ent;
+	while ((ent = readdir(dir))) {
+		if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
+			continue;
+		}
+
+		if (hashmap_get(preserved, ent->d_name) != NULL) continue;
+
+		char target[PATH_MAX];
+		sprintf(target, "%s/%s", path, ent->d_name);
+		if (!rmentry(target)) {
+			closedir(dir);
+			return -1;
+		}
+		removed++;
+	}
+
+	closedir(dir);
+	return removed;
+}
+
+bool
+filesync(const char *restrict srcpath, const char *restrict dstpath,
+		struct hashmap *preserved)
 {
 	int fdsrc;
 	struct stat stsrc;
+	struct vector *own = NULL;
+	bool cleanup = false;
 
 	fdsrc = open(srcpath, O_RDONLY);
-	if (fdsrc < 0) return false;
-	if (fstat(fdsrc, &stsrc)) goto dir_error;
+	if (fdsrc < 0) {
+		log_printl_errno(LOG_ERROR, "Failed to open %s", srcpath);
+		return false;
+	}
+	if (fstat(fdsrc, &stsrc)) {
+		log_printl_errno(LOG_ERROR, "Failed to stat %s", srcpath);
+		goto dir_error;
+	}
 
 	if (S_ISDIR(stsrc.st_mode)) {
 		if (mkdir(dstpath, 0755)) {
-			if (errno != EEXIST) goto dir_error;
-			if (stat(dstpath, &stsrc)) goto dir_error;
+			if (errno != EEXIST) {
+				log_printl_errno(LOG_ERROR, "Failed to create directory %s",
+						dstpath);
+				goto dir_error;
+			}
+			if (stat(dstpath, &stsrc)) {
+				log_printl_errno(LOG_ERROR, "Failed to stat %s", dstpath);
+				goto dir_error;
+			}
 			if (!S_ISDIR(stsrc.st_mode)){
+				log_printl(LOG_ERROR, "%s is not a directory", dstpath);
 				errno = ENOTDIR;
 				goto dir_error;
 			}
+			/* We only need to cleanup if the dir already existed */
+			cleanup = true;
+		}
+
+		if (cleanup) {
+			if (preserved == NULL) {
+				preserved = hashmap_new();
+			} else {
+				own = vector_new(32);
+			}
 		}
+
 		DIR *dir = fdopendir(fdsrc);
-		if (dir == NULL) goto dir_error;
+		if (dir == NULL) {
+			log_printl_errno(LOG_ERROR, "Failed to open directory %s", dstpath);
+			goto dir_error;
+		}
+
 		struct dirent *ent;
 		while ((ent = readdir(dir))) {
 			if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
@@ -179,37 +285,71 @@ filesync(const char *restrict srcpath, const char *restrict dstpath)
 			char entsrc[PATH_MAX], entdst[PATH_MAX];
 			sprintf(entsrc, "%s/%s", srcpath, ent->d_name);
 			sprintf(entdst, "%s/%s", dstpath, ent->d_name);
-			if (filesync(entsrc, entdst)) {
+			if (cleanup) {
+				char *name = strdup(ent->d_name);
+				hashmap_insert(preserved, name, name);
+				if (own) {
+					vector_push(own, name);
+				}
+			}
+			if (!filesync(entsrc, entdst, NULL)) {
 				closedir(dir);
-				goto dir_error;
+				return false;
 			}
 		}
-		closedir(dir);
 
-		goto dir_success;
+		if (cleanup) {
+			rmextra(dstpath, preserved);
+			if (own) {
+				for (size_t i = 0; i < own->size; i++) {
+					free(own->values[i]);
+				}
+				vector_free(own);
+			} else {
+				hashmap_destroy(preserved, hm_destroy_callback);
+			}
+		}
+
+		closedir(dir);
+		return true;
 	}
 
 	int fddst, uptodate;
 	if ((uptodate = file_is_uptodate(dstpath, &stsrc.st_mtim)) > 0) {
-		goto dir_success;
+		goto success;
 	} else if (uptodate < 0) {
 		goto dir_error;
 	}
 	fddst = open(dstpath, O_CREAT | O_WRONLY | O_TRUNC, 0644);
-	if (fddst < 0) goto dir_error;
+	if (fddst < 0) {
+		log_printl_errno(LOG_ERROR, "Failed to open/create %s", dstpath);
+		goto dir_error;
+	}
 
 #ifdef __linux__
 	ssize_t nwrote = sendfile(fddst, fdsrc, NULL, stsrc.st_size);
-	if (nwrote != stsrc.st_size) goto copy_error;
+	if (nwrote != stsrc.st_size) {
+		log_printl_errno(LOG_ERROR, "Failed to copy %s (wrote %lu/%lu bytes)", 
+				dstpath, nwrote, stsrc.st_size);
+		goto copy_error;
+	}
 #else
 	char buf[BUFSIZE];
 	ssize_t nread;
 
 	while ((nread = read(fdsrc, buf, BUFSIZE)) > 0) {
 		ssize_t nwrote = write(fddst, buf, nread);
-		if (nread != nwrote) goto copy_error;
+		if (nread != nwrote) {
+			log_printl_errno(LOG_ERROR, "Failed to copy %s "
+					"(buffer wrote %lu/%lu bytes)", 
+					dstpath, nwrote, nread);
+			goto copy_error;
+		}
+	}
+	if (nread < 0) {
+		log_printl_errno(LOG_ERROR, "Failed to copy %s");
+		goto copy_error;
 	}
-	if (nread < 0) goto copy_error;
 #endif
 
 	struct timespec tms[] = {
@@ -221,7 +361,7 @@ filesync(const char *restrict srcpath, const char *restrict dstpath)
 	log_printl(LOG_DETAIL, "Copied %s", srcpath);
 
 	close(fddst);
-dir_success:
+success:
 	close(fdsrc);
 	return true;
 copy_error:
diff --git a/src/render.c b/src/render.c
index acd1de2..79fb543 100644
--- a/src/render.c
+++ b/src/render.c
@@ -84,11 +84,9 @@ years_destroy(struct vector *years)
 	vector_free(years);
 }
 
-static bool
-albums_walk(struct bstnode *node, void *data)
+bool
+render_set_album_vars(struct render *r, struct album *album)
 {
-	struct album *album = node->value;
-	struct render *r = data;
 
 	hashmap_insert(album->map, "title", album->config->title);
 	hashmap_insert(album->map, "desc", album->config->desc);
@@ -213,7 +211,7 @@ render_init(struct render *r, const char *root, struct site_config *conf,
 	r->years = vector_new(8);
 	r->albums = vector_new(64);
 
-	r->common_vars = hashmap_new_with_cap(8);
+	r->common_vars = hashmap_new_with_cap(16);
 	hashmap_insert(r->common_vars, "title", conf->title);
 	if (strlen(conf->base_url) == 0) {
 		hashmap_insert(r->common_vars, "index", "/");
@@ -221,7 +219,7 @@ render_init(struct render *r, const char *root, struct site_config *conf,
 		hashmap_insert(r->common_vars, "index", conf->base_url);
 	}
 
-	bstree_inorder_walk(albums->root, albums_walk, (void *)r);
+	//bstree_inorder_walk(albums->root, albums_walk, (void *)r);
 
 cleanup:
 	free(tmplpath);
diff --git a/src/site.c b/src/site.c
index 5fa40c4..6fe8d85 100644
--- a/src/site.c
+++ b/src/site.c
@@ -9,12 +9,15 @@
 #include "fs.h"
 #include "log.h"
 #include "hashmap.h"
+#include "render.h"
 
 /* TODO: Probably shouldn't use PATH_MAX, but i'll leave it for now */
 /* TODO: handle error cases for paths that are too long */
 
 #define THUMB_SUFFIX "_thumb"
 
+static const char *index_html = "index.html";
+
 static bool
 wand_passfail(MagickWand *wand, MagickPassFail status)
 {
@@ -80,6 +83,8 @@ images_walk(struct bstnode *node, void *data)
 	struct site *site = data;
 	struct image *image = node->value;
 	struct stat dstat;
+	char htmlpath[PATH_MAX];
+	const char *base = rbasename(image->dst);
 
 	log_printl(LOG_DEBUG, "Image: %s, datetime %s", image->basename, 
 			image->datestr);
@@ -101,8 +106,8 @@ images_walk(struct bstnode *node, void *data)
 		MagickRemoveImage(site->wand);
 	}
 
-	char htmlpath[PATH_MAX];
-	joinpathb(htmlpath, image->dst, "index.html");
+	joinpathb(htmlpath, image->dst, index_html);
+	hashmap_insert(image->album->preserved, base, (char *)base);
 	return render_make_image(&site->render, htmlpath, image);
 magick_fail:
 	return false;
@@ -116,12 +121,27 @@ albums_walk(struct bstnode *node, void *data)
 	struct stat dstat;
 	if (!nmkdir(album->slug, &dstat, site->dry_run)) return false;
 
+	if (!site->dry_run) {
+		hashmap_insert(site->album_dirs, album->slug, (char *)album->slug);
+		if (!render_set_album_vars(&site->render, album)) return false;
+	}
+
 	char htmlpath[PATH_MAX];
-	joinpathb(htmlpath, album->slug, "index.html");
+	joinpathb(htmlpath, album->slug, index_html);
 	if (!render_make_album(&site->render, htmlpath, album)) return false;
 
 	log_printl(LOG_DEBUG, "Album: %s, datetime %s", album->slug, album->datestr);
-	return bstree_inorder_walk(album->images->root, images_walk, site);
+	if (!bstree_inorder_walk(album->images->root, images_walk, site)) {
+		return false;
+	}
+
+	hashmap_insert(album->preserved, index_html, (char *)index_html);
+	if (rmextra(album->slug, album->preserved) < 0) {
+		log_printl_errno(LOG_ERROR, 
+					"Something happened while deleting extraneous files");
+	}
+
+	return true;
 }
 
 /*
@@ -201,24 +221,35 @@ site_build(struct site *site)
 
 	if (!nmkdir(site->output_dir, &dstat, false)) return false;
 
-	joinpathb(staticp, site->root_dir, STATICDIR);
-	if (!filesync(staticp, site->output_dir) && errno != ENOENT) {
-		log_printl_errno(LOG_FATAL, "Can't copy static files");
-		return false;
-	}
-
 	if (chdir(site->output_dir)) {
 		log_printl_errno(LOG_FATAL, "Can't change to directory %s",
 				site->output_dir);
 		return false;
 	}
 
-	if (!render_make_index(&site->render, "index.html")) return false;
-
 	if (!bstree_inorder_walk(site->albums->root, albums_walk, (void *)site)) {
 		return false;
 	}
 
+	if (!render_make_index(&site->render, index_html)) return false;
+	hashmap_insert(site->album_dirs, index_html, (char *)index_html);
+
+	joinpathb(staticp, site->root_dir, STATICDIR);
+	if (stat(staticp, &dstat)) {
+		if (errno != ENOENT) {
+			log_printl_errno(LOG_FATAL, "Couldn't read static dir");
+			return false;
+		}
+		if (rmextra(site->output_dir, site->album_dirs) < 0) {
+			log_printl_errno(LOG_ERROR,
+					"Something happened while deleting extraneous files");
+		}
+	} else if (!site->dry_run 
+			&& !filesync(staticp, site->output_dir, site->album_dirs)) {
+		log_printl(LOG_FATAL, "Can't copy static files");
+		return false;
+	}
+
 	chdir(site->root_dir);
 	return true;
 }
@@ -256,6 +287,9 @@ site_init(struct site *site)
 	site->rel_content_dir = strlen(site->root_dir) + 1;
 	InitializeMagick(NULL);
 	site->wand = NewMagickWand();
+	if (!site->dry_run) {
+		site->album_dirs = hashmap_new();
+	}
 	site->render.dry_run = site->dry_run;
 
 	return true;
@@ -273,6 +307,7 @@ site_deinit(struct site *site)
 		DestroyMagick();
 	}
 	if (!site->dry_run) {
+		hashmap_free(site->album_dirs);
 		render_deinit(&site->render);
 	}
 }
diff --git a/unja b/unja
-Subproject 8a5405629f7dcbc2504ac55f57775180a011b84
+Subproject bf330c19cd3813da8a9c0ae4f48316747f3fa93