From 91b4034fff22cd902dd1c9785702423124e80e2b Mon Sep 17 00:00:00 2001 From: Jeremy Knope Date: Wed, 24 Dec 2025 19:09:18 -0500 Subject: [PATCH 1/5] fix: catch & error if missing uid or email from OIDC fixes #31 --- .../controllers/oidc_controller.ex | 17 +++++++++++++++++ mise.toml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/fuzzy_catalog_web/controllers/oidc_controller.ex b/lib/fuzzy_catalog_web/controllers/oidc_controller.ex index eeffc5e..cf539ce 100644 --- a/lib/fuzzy_catalog_web/controllers/oidc_controller.ex +++ b/lib/fuzzy_catalog_web/controllers/oidc_controller.ex @@ -155,6 +155,23 @@ defmodule FuzzyCatalogWeb.OIDCController do "Handling successful OIDC auth for email: #{email}, provider_uid: #{provider_uid}" ) + if is_nil(email) || is_nil(provider_uid) do + Logger.error( + "OIDC provider returned nil values for email or uid - user_info: #{inspect(user_info)}" + ) + + conn + |> put_flash( + :error, + "Authentication failed: required information not provided by the authentication service. Please contact support." + ) + |> redirect(to: ~p"/users/log-in") + else + handle_user_creation(conn, email, provider, provider_uid, token) + end + end + + defp handle_user_creation(conn, email, provider, provider_uid, token) do case find_or_create_user(email, provider, provider_uid, token) do {:ok, user} -> Logger.info("OIDC user login successful for user ID: #{user.id}") diff --git a/mise.toml b/mise.toml index bc39053..fc02d1d 100644 --- a/mise.toml +++ b/mise.toml @@ -1,4 +1,4 @@ [tools] -erlang = "27" +erlang = "28" node = "22" elixir = "1.18" From a9689578c04b7624a02cc5f2f405acb6aaa378ae Mon Sep 17 00:00:00 2001 From: Jeremy Knope Date: Wed, 24 Dec 2025 19:12:48 -0500 Subject: [PATCH 2/5] chore: some formatting --- lib/fuzzy_catalog/title_utils.ex | 56 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/lib/fuzzy_catalog/title_utils.ex b/lib/fuzzy_catalog/title_utils.ex index 3a2fb3d..7f88074 100644 --- a/lib/fuzzy_catalog/title_utils.ex +++ b/lib/fuzzy_catalog/title_utils.ex @@ -5,35 +5,37 @@ defmodule FuzzyCatalog.TitleUtils do Provides normalization and matching functions for book titles. """ - @common_postfixes [ - # Audio-specific postfixes - "(Unabridged)", - "(Abridged)", - "(Audio CD)", - "(Audiobook)", + defp common_postfixes do + [ + # Audio-specific postfixes + "(Unabridged)", + "(Abridged)", + "(Audio CD)", + "(Audiobook)", - # Format-specific postfixes - "(Paperback)", - "(Hardcover)", - "(Kindle Edition)", - "(eBook)", - "(Digital)", + # Format-specific postfixes + "(Paperback)", + "(Hardcover)", + "(Kindle Edition)", + "(eBook)", + "(Digital)", - # Edition-specific postfixes - "(1st Edition)", - "(2nd Edition)", - "(3rd Edition)", - "(First Edition)", - "(Second Edition)", - "(Third Edition)", - "(Revised Edition)", - "(Updated Edition)", - "(Anniversary Edition)", + # Edition-specific postfixes + "(1st Edition)", + "(2nd Edition)", + "(3rd Edition)", + "(First Edition)", + "(Second Edition)", + "(Third Edition)", + "(Revised Edition)", + "(Updated Edition)", + "(Anniversary Edition)", - # Generic edition markers - ~r/\(\d+(?:st|nd|rd|th) Edition\)/, - ~r/\(.*? Edition\)/ - ] + # Generic edition markers + ~r/\(\d+(?:st|nd|rd|th) Edition\)/, + ~r/\(.*? Edition\)/ + ] + end @doc """ Normalizes a title by removing common postfixes that don't affect the core title. @@ -66,7 +68,7 @@ defmodule FuzzyCatalog.TitleUtils do def normalize_title(nil), do: nil defp remove_common_postfixes(title) do - Enum.reduce(@common_postfixes, title, fn postfix, acc -> + Enum.reduce(common_postfixes(), title, fn postfix, acc -> case postfix do %Regex{} = regex -> String.replace(acc, regex, "") From 9815bf6bcd11cb604f959e4f04c1b9a70e4ad926 Mon Sep 17 00:00:00 2001 From: Jeremy Knope Date: Wed, 24 Dec 2025 19:31:50 -0500 Subject: [PATCH 3/5] fix: exporter just breaking on attempt to export fixes #30 --- lib/fuzzy_catalog/import_export.ex | 4 +- lib/fuzzy_catalog/import_export/exporter.ex | 70 ++++++++++++------- lib/fuzzy_catalog/import_export/job.ex | 50 +++++++++++-- .../controllers/import_export_controller.ex | 50 ++++++++++--- .../import_export_html/index.html.heex | 2 +- .../import_export_html/show.html.heex | 4 +- 6 files changed, 134 insertions(+), 46 deletions(-) diff --git a/lib/fuzzy_catalog/import_export.ex b/lib/fuzzy_catalog/import_export.ex index a936890..e0dfc96 100644 --- a/lib/fuzzy_catalog/import_export.ex +++ b/lib/fuzzy_catalog/import_export.ex @@ -95,7 +95,7 @@ defmodule FuzzyCatalog.ImportExport do """ def create_export_job(%User{} = user, attrs \\ %{}) do %Job{} - |> Job.export_changeset(Map.put(attrs, :user_id, user.id)) + |> Job.export_changeset(Map.put(attrs, "user_id", user.id)) |> Repo.insert() end @@ -113,7 +113,7 @@ defmodule FuzzyCatalog.ImportExport do """ def create_import_job(%User{} = user, attrs \\ %{}) do %Job{} - |> Job.import_changeset(Map.put(attrs, :user_id, user.id)) + |> Job.import_changeset(Map.put(attrs, "user_id", user.id)) |> Repo.insert() end diff --git a/lib/fuzzy_catalog/import_export/exporter.ex b/lib/fuzzy_catalog/import_export/exporter.ex index 60fe471..89c987b 100644 --- a/lib/fuzzy_catalog/import_export/exporter.ex +++ b/lib/fuzzy_catalog/import_export/exporter.ex @@ -13,20 +13,32 @@ defmodule FuzzyCatalog.ImportExport.Exporter do @supported_formats ["json", "csv"] + require Logger + @doc """ Performs an export job asynchronously. """ def perform_export(%Job{} = job) do + Logger.info("Starting export for job ##{job.id}") + case update_job_status(job, "processing") do {:ok, job} -> try do - export_data(job) + result = export_data(job) + Logger.info("Export completed for job ##{job.id}: #{inspect(result)}") + result rescue error -> + Logger.error("Export failed for job ##{job.id}: #{Exception.message(error)}") + Logger.error("Stacktrace: #{Exception.format_stacktrace(__STACKTRACE__)}") ImportExport.fail_job(job, "Export failed: #{Exception.message(error)}") end - {:error, _changeset} -> + {:error, changeset} -> + Logger.error( + "Failed to update job status for job ##{job.id}: #{inspect(changeset.errors)}" + ) + {:error, "Failed to update job status"} end end @@ -34,30 +46,38 @@ defmodule FuzzyCatalog.ImportExport.Exporter do defp export_data(%Job{} = job) do filters = job.filters || %{} format = Map.get(filters, "format", "json") + # Handle empty string format + format = if format == "", do: "json", else: format + + Logger.info("Export format: #{format}, filters: #{inspect(filters)}") unless format in @supported_formats do + Logger.error("Unsupported format: #{format}") ImportExport.fail_job(job, "Unsupported format: #{format}") - end - - # Get filtered data - query_result = build_export_query(filters) - collection_items = Repo.all(query_result.query) - total_items = length(collection_items) - - # Update job with total count - ImportExport.update_job_progress(job, %{ - total_items: total_items, - processed_items: 0, - progress: 0 - }) - - # Process data in chunks for large exports - chunk_size = 100 - chunks = Enum.chunk_every(collection_items, chunk_size) - - case format do - "json" -> export_to_json(job, chunks, total_items) - "csv" -> export_to_csv(job, chunks, total_items) + {:error, "Unsupported format"} + else + # Get filtered data + query_result = build_export_query(filters) + collection_items = Repo.all(query_result.query) + total_items = length(collection_items) + + Logger.info("Found #{total_items} collection items to export") + + # Update job with total count + ImportExport.update_job_progress(job, %{ + total_items: total_items, + processed_items: 0, + progress: 0 + }) + + # Process data in chunks for large exports + chunk_size = 100 + chunks = Enum.chunk_every(collection_items, chunk_size) + + case format do + "json" -> export_to_json(job, chunks, total_items) + "csv" -> export_to_csv(job, chunks, total_items) + end end end @@ -201,7 +221,7 @@ defmodule FuzzyCatalog.ImportExport.Exporter do end new_processed = processed_count + length(chunk) - progress = round(new_processed / total_items * 100) + progress = if total_items > 0, do: round(new_processed / total_items * 100), else: 100 ImportExport.update_job_progress(job, %{ processed_items: new_processed, @@ -289,7 +309,7 @@ defmodule FuzzyCatalog.ImportExport.Exporter do IO.write(file, Enum.join(csv_rows, "\n") <> "\n") new_processed = processed_count + length(chunk) - progress = round(new_processed / total_items * 100) + progress = if total_items > 0, do: round(new_processed / total_items * 100), else: 100 ImportExport.update_job_progress(job, %{ processed_items: new_processed, diff --git a/lib/fuzzy_catalog/import_export/job.ex b/lib/fuzzy_catalog/import_export/job.ex index 6a878c7..17c207e 100644 --- a/lib/fuzzy_catalog/import_export/job.ex +++ b/lib/fuzzy_catalog/import_export/job.ex @@ -58,9 +58,30 @@ defmodule FuzzyCatalog.ImportExport.Job do """ def export_changeset(job, attrs) do job - |> changeset(attrs) + |> cast(attrs, [ + :file_path, + :file_name, + :file_size, + :progress, + :total_items, + :processed_items, + :error_message, + :filters, + :user_id + ]) |> put_change(:type, "export") - |> put_change(:expires_at, DateTime.utc_now() |> DateTime.add(7, :day)) + |> put_change(:status, "pending") + |> put_change( + :expires_at, + DateTime.utc_now() |> DateTime.add(7, :day) |> DateTime.truncate(:second) + ) + |> validate_required([:type, :status, :user_id]) + |> validate_inclusion(:type, @type_values) + |> validate_inclusion(:status, @status_values) + |> validate_number(:progress, greater_than_or_equal_to: 0, less_than_or_equal_to: 100) + |> validate_number(:processed_items, greater_than_or_equal_to: 0) + |> validate_number(:total_items, greater_than_or_equal_to: 0) + |> foreign_key_constraint(:user_id) end @doc """ @@ -68,9 +89,26 @@ defmodule FuzzyCatalog.ImportExport.Job do """ def import_changeset(job, attrs) do job - |> changeset(attrs) + |> cast(attrs, [ + :file_path, + :file_name, + :file_size, + :progress, + :total_items, + :processed_items, + :error_message, + :filters, + :user_id + ]) |> put_change(:type, "import") - |> validate_required([:file_path, :file_name]) + |> put_change(:status, "pending") + |> validate_required([:type, :status, :user_id, :file_path, :file_name]) + |> validate_inclusion(:type, @type_values) + |> validate_inclusion(:status, @status_values) + |> validate_number(:progress, greater_than_or_equal_to: 0, less_than_or_equal_to: 100) + |> validate_number(:processed_items, greater_than_or_equal_to: 0) + |> validate_number(:total_items, greater_than_or_equal_to: 0) + |> foreign_key_constraint(:user_id) end @doc """ @@ -90,7 +128,7 @@ defmodule FuzzyCatalog.ImportExport.Job do job |> cast(attrs, [:file_path, :file_name, :file_size, :error_message]) |> put_change(:status, "completed") - |> put_change(:completed_at, DateTime.utc_now()) + |> put_change(:completed_at, DateTime.utc_now() |> DateTime.truncate(:second)) |> put_change(:progress, 100) end @@ -101,7 +139,7 @@ defmodule FuzzyCatalog.ImportExport.Job do job |> cast(%{error_message: error_message}, [:error_message]) |> put_change(:status, "failed") - |> put_change(:completed_at, DateTime.utc_now()) + |> put_change(:completed_at, DateTime.utc_now() |> DateTime.truncate(:second)) end @doc """ diff --git a/lib/fuzzy_catalog_web/controllers/import_export_controller.ex b/lib/fuzzy_catalog_web/controllers/import_export_controller.ex index ebb0ce8..7d16a07 100644 --- a/lib/fuzzy_catalog_web/controllers/import_export_controller.ex +++ b/lib/fuzzy_catalog_web/controllers/import_export_controller.ex @@ -3,7 +3,6 @@ defmodule FuzzyCatalogWeb.ImportExportController do alias FuzzyCatalog.ImportExport alias FuzzyCatalog.ImportExport.{Job, Exporter} - alias FuzzyCatalog.Storage def index(conn, _params) do current_user = conn.assigns.current_scope.user @@ -20,10 +19,14 @@ defmodule FuzzyCatalogWeb.ImportExportController do end def create_export(conn, %{"job" => job_params}) do + require Logger current_user = conn.assigns.current_scope.user + Logger.info("Creating export job with params: #{inspect(job_params)}") + case ImportExport.create_export_job(current_user, job_params) do {:ok, job} -> + Logger.info("Export job created successfully, spawning export process for job ##{job.id}") # Start export process asynchronously spawn(fn -> Exporter.perform_export(job) end) @@ -35,6 +38,7 @@ defmodule FuzzyCatalogWeb.ImportExportController do |> redirect(to: ~p"/admin/import-export") {:error, changeset} -> + Logger.error("Failed to create export job: #{inspect(changeset.errors)}") filter_options = Exporter.available_filters() render(conn, :new_export, changeset: changeset, filter_options: filter_options) end @@ -148,14 +152,31 @@ defmodule FuzzyCatalogWeb.ImportExportController do |> put_flash(:error, "Export file not available") |> redirect(to: ~p"/admin/import-export") else - case Storage.get_file_url(job.file_path) do - {:ok, file_url} -> - redirect(conn, external: file_url) + # Build the full file path + base_path = + Application.get_env(:fuzzy_catalog, :storage, []) + |> Keyword.get(:local, []) + |> Keyword.get(:base_path, "priv/static/uploads") + + full_path = Path.join(base_path, job.file_path) + + if File.exists?(full_path) do + # Determine content type based on file extension + content_type = + case Path.extname(job.file_name) do + ".json" -> "application/json" + ".csv" -> "text/csv" + _ -> "application/octet-stream" + end - {:error, _reason} -> - conn - |> put_flash(:error, "Export file not found") - |> redirect(to: ~p"/admin/import-export") + conn + |> put_resp_content_type(content_type) + |> put_resp_header("content-disposition", "attachment; filename=\"#{job.file_name}\"") + |> send_file(200, full_path) + else + conn + |> put_flash(:error, "Export file not found") + |> redirect(to: ~p"/admin/import-export") end end end @@ -167,8 +188,17 @@ defmodule FuzzyCatalogWeb.ImportExportController do case ImportExport.delete_job(job) do {:ok, _job} -> # Clean up associated file - if job.file_path && File.exists?(job.file_path) do - File.rm(job.file_path) + if job.file_path do + base_path = + Application.get_env(:fuzzy_catalog, :storage, []) + |> Keyword.get(:local, []) + |> Keyword.get(:base_path, "priv/static/uploads") + + full_path = Path.join(base_path, job.file_path) + + if File.exists?(full_path) do + File.rm(full_path) + end end conn diff --git a/lib/fuzzy_catalog_web/controllers/import_export_html/index.html.heex b/lib/fuzzy_catalog_web/controllers/import_export_html/index.html.heex index 13a4aa2..4b3604e 100644 --- a/lib/fuzzy_catalog_web/controllers/import_export_html/index.html.heex +++ b/lib/fuzzy_catalog_web/controllers/import_export_html/index.html.heex @@ -100,7 +100,7 @@ max="100" > - <%= if job.processed_items and job.total_items do %> + <%= if not is_nil(job.processed_items) and not is_nil(job.total_items) do %>

{job.processed_items} of {job.total_items} items processed

diff --git a/lib/fuzzy_catalog_web/controllers/import_export_html/show.html.heex b/lib/fuzzy_catalog_web/controllers/import_export_html/show.html.heex index 62eb840..86f8b36 100644 --- a/lib/fuzzy_catalog_web/controllers/import_export_html/show.html.heex +++ b/lib/fuzzy_catalog_web/controllers/import_export_html/show.html.heex @@ -62,7 +62,7 @@ {@job.progress || 0}% - <%= if @job.processed_items and @job.total_items do %> + <%= if not is_nil(@job.processed_items) and not is_nil(@job.total_items) do %>

{@job.processed_items} of {@job.total_items} items processed

@@ -110,7 +110,7 @@ <% end %> - <%= if @job.filters and map_size(@job.filters) > 0 do %> + <%= if not is_nil(@job.filters) and map_size(@job.filters) > 0 do %>
Filters Applied
From b9a72278d1085466d6bb80599d932cd7d38ee08c Mon Sep 17 00:00:00 2001 From: Jeremy Knope Date: Wed, 24 Dec 2025 20:06:05 -0500 Subject: [PATCH 4/5] fix: import page rendering poorly in dark mode --- .../import_export_html/new_import.html.heex | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/fuzzy_catalog_web/controllers/import_export_html/new_import.html.heex b/lib/fuzzy_catalog_web/controllers/import_export_html/new_import.html.heex index 1594ae0..0235075 100644 --- a/lib/fuzzy_catalog_web/controllers/import_export_html/new_import.html.heex +++ b/lib/fuzzy_catalog_web/controllers/import_export_html/new_import.html.heex @@ -82,14 +82,17 @@
-
-

Import Requirements

+
+

Import Requirements

-

JSON Format

-

+

JSON Format

+

Must contain an "items" array with book objects. Each book should have at minimum a "title" field.

- 
+        
 {
   "items": [
     {
@@ -106,18 +109,18 @@
-

CSV Format

-

+

CSV Format

+

Must include "title" and "author" columns at minimum. Additional columns will be imported if they match book fields.

- 
+        
 title,author,isbn13,media_type,genre
 "Book Title","Author Name","9781234567890","book","Fiction"
-
-

Import Process

+
+

Import Process

  • Files are validated and you'll see a preview before importing
  • Duplicate books are detected by ISBN or title/author combination
    • From 0a15f84271c65a444297ce68d3568bd528dd2a37 Mon Sep 17 00:00:00 2001 From: Jeremy Knope Date: Fri, 2 Jan 2026 19:47:15 -0500 Subject: [PATCH 5/5] fix: OIDC login failing with Authelia or really any others that don't automatically include email in callback/token --- .../controllers/oidc_controller.ex | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/lib/fuzzy_catalog_web/controllers/oidc_controller.ex b/lib/fuzzy_catalog_web/controllers/oidc_controller.ex index cf539ce..e94e073 100644 --- a/lib/fuzzy_catalog_web/controllers/oidc_controller.ex +++ b/lib/fuzzy_catalog_web/controllers/oidc_controller.ex @@ -9,13 +9,8 @@ defmodule FuzzyCatalogWeb.OIDCController do def authorize(conn, _params) do config = Application.get_env(:fuzzy_catalog, :oidc) - Logger.info("OIDC authorize initiated with config: #{inspect(config, pretty: true)}") - case Assent.Strategy.OIDC.authorize_url(config) do {:ok, %{url: url, session_params: session_params}} -> - Logger.info("OIDC authorize successful, redirecting to: #{url}") - Logger.info("Session params to store: #{inspect(session_params, pretty: true)}") - conn |> put_session(:oidc_state, session_params["state"]) |> put_session(:oidc_nonce, session_params["nonce"]) @@ -43,14 +38,6 @@ defmodule FuzzyCatalogWeb.OIDCController do nonce = get_session(conn, :oidc_nonce) stored_session_params = get_session(conn, :oidc_session_params) - Logger.info( - "OIDC callback received with params: #{inspect(Map.drop(params, ["code"]), pretty: true)}" - ) - - Logger.info("Retrieved session state: #{inspect(state)}") - Logger.info("Retrieved session nonce: #{inspect(nonce)}") - Logger.info("Stored session params: #{inspect(stored_session_params, pretty: true)}") - # Use stored session params if individual values are missing, or fallback to params session_params = cond do @@ -84,8 +71,6 @@ defmodule FuzzyCatalogWeb.OIDCController do %{} end - Logger.info("Final session params for callback: #{inspect(session_params, pretty: true)}") - # Check if we have required state parameter state_value = session_params["state"] @@ -109,13 +94,19 @@ defmodule FuzzyCatalogWeb.OIDCController do # Merge session_params into config for Assent config_with_session = Keyword.put(config, :session_params, session_params_atoms) - Logger.info( - "Config with session params (atom keys): #{inspect(config_with_session, pretty: true)}" - ) - case Assent.Strategy.OIDC.callback(config_with_session, params) do {:ok, %{user: user_info, token: token}} -> - Logger.info("OIDC callback successful for user: #{inspect(user_info["email"])}") + # If user_info is sparse, try to fetch from userinfo endpoint + user_info = + if is_nil(user_info["email"]) || is_nil(user_info["mail"]) do + case Assent.Strategy.OIDC.fetch_userinfo(config_with_session, token) do + {:ok, fetched_user_info} -> Map.merge(user_info, fetched_user_info) + {:error, _error} -> user_info + end + else + user_info + end + handle_successful_auth(conn, user_info, token) {:error, error} -> @@ -147,14 +138,11 @@ defmodule FuzzyCatalogWeb.OIDCController do end defp handle_successful_auth(conn, user_info, token) do - email = user_info["email"] + # Try multiple common claims for email as some providers use 'mail' or others + email = user_info["email"] || user_info["mail"] provider_uid = user_info["sub"] provider = "oidc" - Logger.info( - "Handling successful OIDC auth for email: #{email}, provider_uid: #{provider_uid}" - ) - if is_nil(email) || is_nil(provider_uid) do Logger.error( "OIDC provider returned nil values for email or uid - user_info: #{inspect(user_info)}"