Add infrastructure to cache images easily

softlion · Jan 13, 2024 · 8aa4d6b · 8aa4d6b
1 parent 7bbd8bc
commit 8aa4d6b
Show file tree

Hide file tree

Showing 7 changed files with 115 additions and 27 deletions.
diff --git a/NotionSharp.ApiClient.Tests/JsonData/AboutThis.full.children.json b/NotionSharp.ApiClient.Tests/JsonData/AboutThis.full.children.json
@@ -510,7 +510,35 @@
     "object": "block"
   },
   {
-    "id": "7919eb3f-cffe-46cb-95d0-8a4f27830a88",
+    "id": "08c7c842-8742-44ef-bcaa-41227dd58100",
+    "parent": {
+      "type": "page_id",
+      "page_id": "c7b44455-3d31-4a5b-b82c-b7e3d85ba585"
+    },
+    "type": "image",
+    "created_time": "2024-01-12T19:14:00+00:00",
+    "created_by": {
+      "id": "ab9257e1-d027-4494-8792-71d90b63dd35",
+      "object": "user"
+    },
+    "last_edited_time": "2024-01-12T19:15:00+00:00",
+    "last_edited_by": {
+      "id": "ab9257e1-d027-4494-8792-71d90b63dd35",
+      "object": "user"
+    },
+    "archived": false,
+    "has_children": false,
+    "image": {
+      "file": {
+        "url": "https://prod-files-secure.s3.us-west-2.amazonaws.com/da89a8e0-9c55-4c13-9966-796e0f0c1bac/04e5e722-c051-4268-b73d-5ddb4b9c69d4/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256\u0026X-Amz-Content-Sha256=UNSIGNED-PAYLOAD\u0026X-Amz-Credential=AKIAT73L2G45HZZMZUHI%2F20240112%2Fus-west-2%2Fs3%2Faws4_request\u0026X-Amz-Date=20240112T191654Z\u0026X-Amz-Expires=3600\u0026X-Amz-Signature=23d76bc0c356a8218af7797966f0758077c8709ef0d68085d99e9ff408ae2e75\u0026X-Amz-SignedHeaders=host\u0026x-id=GetObject",
+        "expiry_time": "2024-01-12T20:16:54.526Z"
+      },
+      "type": "file"
+    },
+    "object": "block"
+  },
+  {
+    "id": "ca8773b0-1dea-4ac3-a693-ebca19da0816",
     "parent": {
       "type": "page_id",
       "page_id": "c7b44455-3d31-4a5b-b82c-b7e3d85ba585"
@@ -521,7 +549,31 @@
       "id": "ab9257e1-d027-4494-8792-71d90b63dd35",
       "object": "user"
     },
-    "last_edited_time": "2020-05-27T10:20:00+00:00",
+    "last_edited_time": "2024-01-12T19:14:00+00:00",
+    "last_edited_by": {
+      "id": "ab9257e1-d027-4494-8792-71d90b63dd35",
+      "object": "user"
+    },
+    "archived": false,
+    "has_children": false,
+    "paragraph": {
+      "rich_text": []
+    },
+    "object": "block"
+  },
+  {
+    "id": "72f80937-724a-4c7e-98cc-f90db4d28ba6",
+    "parent": {
+      "type": "page_id",
+      "page_id": "c7b44455-3d31-4a5b-b82c-b7e3d85ba585"
+    },
+    "type": "paragraph",
+    "created_time": "2020-04-30T13:39:00+00:00",
+    "created_by": {
+      "id": "ab9257e1-d027-4494-8792-71d90b63dd35",
+      "object": "user"
+    },
+    "last_edited_time": "2024-01-12T19:14:00+00:00",
     "last_edited_by": {
       "id": "ab9257e1-d027-4494-8792-71d90b63dd35",
       "object": "user"
@@ -1024,8 +1076,8 @@
             "has_children": false,
             "image": {
               "file": {
-                "url": "https://prod-files-secure.s3.us-west-2.amazonaws.com/da89a8e0-9c55-4c13-9966-796e0f0c1bac/3b632032-656f-4e87-87b7-b95bcb786f9d/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256\u0026X-Amz-Content-Sha256=UNSIGNED-PAYLOAD\u0026X-Amz-Credential=AKIAT73L2G45HZZMZUHI%2F20240112%2Fus-west-2%2Fs3%2Faws4_request\u0026X-Amz-Date=20240112T131938Z\u0026X-Amz-Expires=3600\u0026X-Amz-Signature=4a7d18333c0e39b278886df1631c04fb15b180fb6eacfe3dee0aabbd8a78a948\u0026X-Amz-SignedHeaders=host\u0026x-id=GetObject",
-                "expiry_time": "2024-01-12T14:19:38.300Z"
+                "url": "https://prod-files-secure.s3.us-west-2.amazonaws.com/da89a8e0-9c55-4c13-9966-796e0f0c1bac/3b632032-656f-4e87-87b7-b95bcb786f9d/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256\u0026X-Amz-Content-Sha256=UNSIGNED-PAYLOAD\u0026X-Amz-Credential=AKIAT73L2G45HZZMZUHI%2F20240112%2Fus-west-2%2Fs3%2Faws4_request\u0026X-Amz-Date=20240112T191656Z\u0026X-Amz-Expires=3600\u0026X-Amz-Signature=bda2b0e78f76744f276eef9714cf94d987685fe7a0d218cce21f2b514a915e01\u0026X-Amz-SignedHeaders=host\u0026x-id=GetObject",
+                "expiry_time": "2024-01-12T20:16:56.810Z"
               },
               "type": "file"
             },

diff --git a/NotionSharp.ApiClient.Tests/JsonData/AboutThis.full.json b/NotionSharp.ApiClient.Tests/JsonData/AboutThis.full.json
@@ -1 +1 @@
-{"id":"c7b44455-3d31-4a5b-b82c-b7e3d85ba585","created_time":"2020-04-30T13:37:00+00:00","last_edited_time":"2024-01-12T12:34:00+00:00","archived":false,"created_by":{"id":"ab9257e1-d027-4494-8792-71d90b63dd35","object":"user"},"last_edited_by":{"id":"ab9257e1-d027-4494-8792-71d90b63dd35","object":"user"},"url":"https://www.notion.so/About-this-c7b444553d314a5bb82cb7e3d85ba585","public_url":"https://wise-spirit-737.notion.site/About-this-c7b444553d314a5bb82cb7e3d85ba585","parent":{"page_id":"18dfbe55-5d7c-416e-9485-7855d4a3949e","type":"page_id"},"properties":{"title":{"title":[{"type":"text","plain_text":"About this","annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"text":{"content":"About this"}}],"id":"title","type":"title"}},"object":"page"}
+{"id":"c7b44455-3d31-4a5b-b82c-b7e3d85ba585","created_time":"2020-04-30T13:37:00+00:00","last_edited_time":"2024-01-12T19:15:00+00:00","archived":false,"created_by":{"id":"ab9257e1-d027-4494-8792-71d90b63dd35","object":"user"},"last_edited_by":{"id":"ab9257e1-d027-4494-8792-71d90b63dd35","object":"user"},"url":"https://www.notion.so/About-this-c7b444553d314a5bb82cb7e3d85ba585","public_url":"https://wise-spirit-737.notion.site/About-this-c7b444553d314a5bb82cb7e3d85ba585","parent":{"page_id":"18dfbe55-5d7c-416e-9485-7855d4a3949e","type":"page_id"},"properties":{"title":{"title":[{"type":"text","plain_text":"About this","annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"text":{"content":"About this"}}],"id":"title","type":"title"}},"object":"page"}
diff --git a/NotionSharp.ApiClient.Tests/NotionSharp.ApiClient.Tests.csproj b/NotionSharp.ApiClient.Tests/NotionSharp.ApiClient.Tests.csproj
@@ -4,6 +4,7 @@
         <TargetFramework>net8.0</TargetFramework>
 
         <IsPackable>false</IsPackable>
+        <Nullable>enable</Nullable>
 
         <UserSecretsId>a04bcd1f-7c68-4874-ac98-81af967e3eb3</UserSecretsId>
     </PropertyGroup>

diff --git a/NotionSharp.ApiClient.Tests/TestNotionBase.cs b/NotionSharp.ApiClient.Tests/TestNotionBase.cs
@@ -4,6 +4,7 @@
 using System.IO.MemoryMappedFiles;
 using System.Linq;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using NotionSharp.ApiClient.Lib;
@@ -210,9 +211,16 @@ public async Task TestGetSyndicationFeed()
         Assert.IsNotNull(pages);
         Assert.AreEqual(1, pages.Count);
 
-        var feed = await session.GetSyndicationFeed(pages[0], new ("https://"));
+        var feed = await session.GetSyndicationFeed(pages[0], new ("https://test.com"), 
+            cacheImage: LocalCacheImage);
+
         Assert.IsNotNull(feed?.Items);
         Assert.AreNotEqual(0, feed.Items.Count());
+
+        static async Task<string?> LocalCacheImage(string imageId, string imageUrl, CancellationToken cancel)
+        {
+            return imageUrl;
+        }
     }
 
     [TestMethod]

diff --git a/NotionSharp.ApiClient/Lib/PublicApi/Model/Block.cs b/NotionSharp.ApiClient/Lib/PublicApi/Model/Block.cs
@@ -128,7 +128,7 @@ public Block()
 
     public BlockChildPage? ChildPage { get; init; }
 
-    public NotionFile? Image { get; init; }
+    public NotionFile? Image { get; set; }
     public NotionFile? File { get; init; }
     public BlockTextAndChildrenAndColor? Quote { get; set; }
     public BlockCallout? Callout { get; init; }

diff --git a/NotionSharp.ApiClient/NotionSessionExtensions.cs b/NotionSharp.ApiClient/NotionSessionExtensions.cs
@@ -293,7 +293,29 @@ public static async Task LoadChildBlocks(this NotionSession session, List<Block>
         }
     }
 
+    /// <summary>
+    /// imageId: unique image ID
+    /// imageUrl: current temporary url
+    /// return: new static url, or null
+    /// </summary>
+    public delegate Task<string?> CacheImageDelegate(string imageId, string imageUrl, CancellationToken cancel);
 
+    /// <summary>
+    /// Cache images contained into the blocks
+    /// </summary>
+    /// <param name="blocks">List of child blocks of a page</param>
+    /// <param name="cacheImage">Your caching system</param>
+    /// <param name="cancel"></param>
+    public static async Task CacheImages(IEnumerable<Block> blocks, CacheImageDelegate cacheImage, CancellationToken cancel = default)
+    {
+        await Parallel.ForEachAsync(blocks.Where(b => b is { Type: BlockTypes.Image, Image.File.Url: not null }), cancel, async (block, localCancel) =>
+        {
+            var newUrl = await cacheImage.Invoke(block.Id, block.Image!.File!.Url, localCancel);
+            if (newUrl != null)
+                block.Image = block.Image with { File = block.Image.File with { Url = newUrl } };
+        });
+    }
+
     /// <summary>
     /// Create a syndication feed from the child pages of this page.
     /// </summary>
@@ -303,6 +325,7 @@ public static async Task LoadChildBlocks(this NotionSession session, List<Block>
     /// <param name="maxItems">max number of child pages to return</param>
     /// <param name="maxBlocks">limit the parsing of each page to the first maxBlocks blocks</param>
     /// <param name="stopBeforeFirstSubHeader">when true, stop parsing a page when a line containing a sub_header is found</param>
+    /// <param name="cacheImage">optionally used to transform the url of images, as notion API returns URLs that expire quickly</param>
     /// <param name="cancel"></param>
     /// <returns>A SyndicationFeed containing one SyndicationItem per child page</returns>
     /// <remarks>
@@ -313,17 +336,17 @@ public static async Task LoadChildBlocks(this NotionSession session, List<Block>
     /// 
     /// Are included only child pages which are not restricted with the "integration".
     /// </remarks>
-    public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession session, Page page, Uri baseUrl, int maxItems = 50, int maxBlocks = 20, bool stopBeforeFirstSubHeader = true, CancellationToken cancel = default)
+    public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession session, Page page, Uri baseUrl, int maxItems = 50, int maxBlocks = 20, bool stopBeforeFirstSubHeader = true, CacheImageDelegate? cacheImage = null, CancellationToken cancel = default)
     {
         var childPages = await session.GetBlockChildren(page.Id, cancel: cancel)
-            .Where(b => b is { Type: BlockTypes.ChildPage, ChildPage: { } }) //2 checks are redundant. We could keep only one.
+            .Where(b => b is { Type: BlockTypes.ChildPage, ChildPage: not null }) //2 checks are redundant. We could keep only one.
             .Take(maxItems)
             .ToListAsync(cancel).ConfigureAwait(false);
 
         if (childPages.Count == 0)
             return new() { LastUpdatedTime = DateTimeOffset.Now };
-                    
-        var feed = await session.GetSyndicationFeed(childPages, baseUrl, maxBlocks, stopBeforeFirstSubHeader, cancel).ConfigureAwait(false);
+
+        var feed = await session.GetSyndicationFeed(childPages, baseUrl, maxBlocks, stopBeforeFirstSubHeader, cacheImage, cancel).ConfigureAwait(false);
 
         var title = page.Title();
 
@@ -332,11 +355,12 @@ public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession
         //feed.Title = new TextSyndicationContent(space.Name);
         //feed.Description = new TextSyndicationContent(space.Domain);
         feed.Id = page.Id;
-        feed.Title = new (title.Title.FirstOrDefault()?.PlainText);
+        feed.Title = new (title?.Title.FirstOrDefault()?.PlainText);
         //feed.Description = ??
         return feed;
     }
 
+
     /// <summary>
     /// Create a syndication feed from a list of page
     /// </summary>
@@ -345,12 +369,13 @@ public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession
     /// <param name="baseUrl">URL of the Notion domain for published pages</param>
     /// <param name="maxBlocks">limits the parsing of each page to the first maxBlocks blocks</param>
     /// <param name="stopBeforeFirstSubHeader">when true, stop parsing a page when a line containing a sub_header is found</param>
+    /// <param name="cacheImage">optionally used to transform the url of images, as notion API returns URLs that expire quickly</param>
     /// <param name="cancel"></param>
     /// <returns>A SyndicationFeed containing one SyndicationItem per page</returns>
     /// <remarks>
     /// The created feed has no title/description
     /// </remarks>
-    public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession session, List<Block> childPages, Uri baseUrl, int maxBlocks = 20, bool stopBeforeFirstSubHeader = true, CancellationToken cancel = default)
+    public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession session, List<Block> childPages, Uri baseUrl, int maxBlocks = 20, bool stopBeforeFirstSubHeader = true, CacheImageDelegate? cacheImage = null, CancellationToken cancel = default)
     {
         var feedItems = new List<SyndicationItem>();
         var htmlRenderer = new HtmlRenderer();
@@ -363,10 +388,8 @@ public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession
             if (page.Type != BlockTypes.ChildPage)
                 throw new ArgumentException("All childPages must be of type BlockTypes.ChildPage", nameof(childPages));
 
-            //get blocks and extract an html content
-            // var blocks = await session.GetBlockChildren(page.Id, cancel: cancel)
-            //     .Take(maxBlocks)
-            //     .ToListAsync(cancel).ConfigureAwait(false);
+            if (cacheImage != null)
+                await CacheImages(page.Children, cacheImage, cancel);
 
             var content = htmlRenderer.GetHtml(page.Children, stopBeforeFirstSubHeader);
             var title = page.ChildPage!.Title;
@@ -376,19 +399,23 @@ public static async Task<SyndicationFeed> GetSyndicationFeed(this NotionSession
             {
                 Id = page.Id,
                 BaseUri = pageUri,
-                Summary = new TextSyndicationContent(content),
+                Summary = new (content),
                 PublishDate = page.CreatedTime,
                 LastUpdatedTime = page.LastEditedTime,
             };
 
-            // Property not yet available in API
-            // if (!String.IsNullOrWhiteSpace(page.Format?.PageIcon))
-            // {
-            //     if(Uri.TryCreate(page.Format.PageIcon, UriKind.Absolute, out _))
-            //         item.AttributeExtensions.Add(new XmlQualifiedName("iconUrl"), pageBlock.Format.PageIcon);
-            //     else
-            //         item.AttributeExtensions.Add(new XmlQualifiedName("iconString"), pageBlock.Format.PageIcon);
-            // }
+
+            var pageIconImageUrl = page.Image?.File?.Url;
+            if (!String.IsNullOrWhiteSpace(pageIconImageUrl))
+            {
+                if (cacheImage != null)
+                    pageIconImageUrl = await cacheImage(page.Id, pageIconImageUrl, cancel) ?? pageIconImageUrl;
+
+                if(Uri.TryCreate(pageIconImageUrl, UriKind.Absolute, out _))
+                    item.AttributeExtensions.Add(new ("iconUrl"), pageIconImageUrl);
+                // else
+                //     item.AttributeExtensions.Add(new XmlQualifiedName("iconString"), pageBlock.Format.PageIcon);
+            }
 
             feedItems.Add(item);
         }

diff --git a/NotionSharp.ApiClient/NotionSharp.ApiClient.csproj b/NotionSharp.ApiClient/NotionSharp.ApiClient.csproj
@@ -19,7 +19,7 @@
     <PropertyGroup>
         <!--  nuget configurable properties -->
         <Version>2.0.0</Version>
-        <VersionSuffix>-pre7</VersionSuffix>
+        <VersionSuffix>-pre8</VersionSuffix>
         <DefineConstants>$(DefineConstants);</DefineConstants>
     </PropertyGroup>