commit 0a1e04d30d73b55dce27bc1ad2d311a424b80cc4 Author: Aytac Kirmizi Date: Wed Feb 1 16:02:30 2023 +0100 **update** initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..eb5c53e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/.idea +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +**/appsettings.json +**/appsettings.*.json +LICENSE +README.md \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..87b6abc --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +bin/ +obj/ +/packages/ +riderModule.iml +/_ReSharper.Caches/ +**/.env +.env \ No newline at end of file diff --git a/.idea/.idea.Linkding.Sync/.idea/.gitignore b/.idea/.idea.Linkding.Sync/.idea/.gitignore new file mode 100644 index 0000000..4470321 --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/.gitignore @@ -0,0 +1,13 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Rider ignored files +/contentModel.xml +/.idea.Linkding.Sync.iml +/modules.xml +/projectSettingsUpdater.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/.idea.Linkding.Sync/.idea/active-tab-highlighter.xml b/.idea/.idea.Linkding.Sync/.idea/active-tab-highlighter.xml new file mode 100644 index 0000000..409aba9 --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/active-tab-highlighter.xml @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/.idea/.idea.Linkding.Sync/.idea/aws.xml b/.idea/.idea.Linkding.Sync/.idea/aws.xml new file mode 100644 index 0000000..b63b642 --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/aws.xml @@ -0,0 +1,11 @@ + + + + + + \ No newline at end of file diff --git a/.idea/.idea.Linkding.Sync/.idea/dbnavigator.xml b/.idea/.idea.Linkding.Sync/.idea/dbnavigator.xml new file mode 100644 index 0000000..da5716d --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/dbnavigator.xml @@ -0,0 +1,414 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/.idea.Linkding.Sync/.idea/encodings.xml b/.idea/.idea.Linkding.Sync/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/.idea.Linkding.Sync/.idea/git_toolbox_prj.xml b/.idea/.idea.Linkding.Sync/.idea/git_toolbox_prj.xml new file mode 100644 index 0000000..02b915b --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/git_toolbox_prj.xml @@ -0,0 +1,15 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/.idea.Linkding.Sync/.idea/indexLayout.xml b/.idea/.idea.Linkding.Sync/.idea/indexLayout.xml new file mode 100644 index 0000000..5ef90b1 --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/indexLayout.xml @@ -0,0 +1,10 @@ + + + + + examples + + + + + \ No newline at end of file diff --git a/.idea/.idea.Linkding.Sync/.idea/vcs.xml b/.idea/.idea.Linkding.Sync/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/.idea.Linkding.Sync/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Dockerfile_Linkding b/Dockerfile_Linkding new file mode 100644 index 0000000..8e2fb1d --- /dev/null +++ b/Dockerfile_Linkding @@ -0,0 +1,19 @@ +FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build +WORKDIR /src +COPY ["src/Linkding/Linkding.csproj", "src/Linkding/"] +RUN dotnet restore "src/Linkding/Linkding.csproj" +COPY . . +WORKDIR "/src/src/Linkding" +RUN dotnet build "Linkding.csproj" -c Release -o /app/build + +FROM build AS publish +RUN dotnet publish "Linkding.csproj" -c Release -o /app/publish + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +RUN mkdir ./data +ENTRYPOINT ["dotnet", "Linkding.dll"] diff --git a/Dockerfile_Wallabag b/Dockerfile_Wallabag new file mode 100644 index 0000000..feb9b80 --- /dev/null +++ b/Dockerfile_Wallabag @@ -0,0 +1,19 @@ +FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build +WORKDIR /src +COPY ["src/Wallabag/Wallabag.csproj", "src/Wallabag/"] +RUN dotnet restore "src/Wallabag/Wallabag.csproj" +COPY . . +WORKDIR "/src/src/Wallabag" +RUN dotnet build "Wallabag.csproj" -c Release -o /app/build + +FROM build AS publish +RUN dotnet publish "Wallabag.csproj" -c Release -o /app/publish + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +RUN mkdir ./data +ENTRYPOINT ["dotnet", "Wallabag.dll"] diff --git a/Linkding.Sync.sln b/Linkding.Sync.sln new file mode 100644 index 0000000..ef25416 --- /dev/null +++ b/Linkding.Sync.sln @@ -0,0 +1,61 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{3167CB7E-6412-49DB-BB42-E91E07B51C5E}" + ProjectSection(SolutionItems) = preProject + README.md = README.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Services", "Services", "{2775FBD9-7954-4A8B-8831-31C7670209A3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Linkding.Client", "src\Services\Linkding.Client\Linkding.Client.csproj", "{1880BB32-4013-45F5-9DB4-6864F9836525}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wallabag.Client", "src\Services\Wallabag.Client\Wallabag.Client.csproj", "{2776E05F-F35A-4421-A792-0F9B0CC48475}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Domain", "Domain", "{586A8F51-1A5D-42B9-8A37-8B2010905EB1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Core", "src\Domain\Core\Core.csproj", "{0EA1E116-6560-4C1D-8EC8-6ACE3F5E1C25}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Infrastructure", "Infrastructure", "{953FCC4E-C759-47DD-B20A-78FBA2E34F35}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wallabag", "src\Wallabag\Wallabag.csproj", "{5EFB26BC-CD34-4DE6-B0E3-9F1E387D4177}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Linkding", "src\Linkding\Linkding.csproj", "{3E78F171-D237-46DF-8A27-DAADE7CA1940}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1880BB32-4013-45F5-9DB4-6864F9836525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1880BB32-4013-45F5-9DB4-6864F9836525}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1880BB32-4013-45F5-9DB4-6864F9836525}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1880BB32-4013-45F5-9DB4-6864F9836525}.Release|Any CPU.Build.0 = Release|Any CPU + {2776E05F-F35A-4421-A792-0F9B0CC48475}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2776E05F-F35A-4421-A792-0F9B0CC48475}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2776E05F-F35A-4421-A792-0F9B0CC48475}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2776E05F-F35A-4421-A792-0F9B0CC48475}.Release|Any CPU.Build.0 = Release|Any CPU + {0EA1E116-6560-4C1D-8EC8-6ACE3F5E1C25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0EA1E116-6560-4C1D-8EC8-6ACE3F5E1C25}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0EA1E116-6560-4C1D-8EC8-6ACE3F5E1C25}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0EA1E116-6560-4C1D-8EC8-6ACE3F5E1C25}.Release|Any CPU.Build.0 = Release|Any CPU + {5EFB26BC-CD34-4DE6-B0E3-9F1E387D4177}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5EFB26BC-CD34-4DE6-B0E3-9F1E387D4177}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5EFB26BC-CD34-4DE6-B0E3-9F1E387D4177}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5EFB26BC-CD34-4DE6-B0E3-9F1E387D4177}.Release|Any CPU.Build.0 = Release|Any CPU + {3E78F171-D237-46DF-8A27-DAADE7CA1940}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3E78F171-D237-46DF-8A27-DAADE7CA1940}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E78F171-D237-46DF-8A27-DAADE7CA1940}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3E78F171-D237-46DF-8A27-DAADE7CA1940}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {2775FBD9-7954-4A8B-8831-31C7670209A3} = {3167CB7E-6412-49DB-BB42-E91E07B51C5E} + {1880BB32-4013-45F5-9DB4-6864F9836525} = {2775FBD9-7954-4A8B-8831-31C7670209A3} + {2776E05F-F35A-4421-A792-0F9B0CC48475} = {2775FBD9-7954-4A8B-8831-31C7670209A3} + {586A8F51-1A5D-42B9-8A37-8B2010905EB1} = {3167CB7E-6412-49DB-BB42-E91E07B51C5E} + {0EA1E116-6560-4C1D-8EC8-6ACE3F5E1C25} = {586A8F51-1A5D-42B9-8A37-8B2010905EB1} + {953FCC4E-C759-47DD-B20A-78FBA2E34F35} = {3167CB7E-6412-49DB-BB42-E91E07B51C5E} + {5EFB26BC-CD34-4DE6-B0E3-9F1E387D4177} = {3167CB7E-6412-49DB-BB42-E91E07B51C5E} + {3E78F171-D237-46DF-8A27-DAADE7CA1940} = {3167CB7E-6412-49DB-BB42-E91E07B51C5E} + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md new file mode 100644 index 0000000..09f489a --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +# Linkding Sync +LinkdingSync is a collection of tools that make life with [Linkding](https://github.com/sissbruecker/linkding) easier. + +One of the workers is for syncing to [Wallabag](https://wallabag.org/en). + +## Getting Started +It is recommended to use the Docker images. Otherwise, a .NET 6 environment is required to customize and build the code. + +## Environment Variables +For the containers to work, the environment variables must be passed. This can be done either directly via the Docker run **-e** switch, via the **environment** settings in a Docker compose definition, or via an environment variable file. + +### WallabagSync +Environment variables for the wallabag worker. + +| Variable | Value | Description | Attention | +|------------------------|-----------|--------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Worker__Intervall | int (>=0) | This value sets the execution schedule in minutes. 1 = every minute, 10 = every 10 minutes (default value 0) | 0 = runs only one time. The container will be stopped after the execution. This method is the preferred way to run the container with a scheduler (e.g. cron) | +| Worker__SyncTag | text | The linkding tag to create the bookmarks in Wallabag. (default value 'readlater') | | +| Linkding__Url | text | URL to the linkding instance | | +| Linkding__Key | text | The linkding application key | [Instructions](https://github.com/sissbruecker/linkding/blob/master/docs/API.md) | +| Wallabag__Url | text | URL to the Wallabag instance | | +| Wallabag__Username | text | Wallabag User Name | | +| Wallabag__Password | text | Wallabag User Password | | +| Wallabag__ClientId | text | Wallabag Client Id | | +| Wallabag__ClientSecret | text | Wallabag Client Secret | | + +### LinkdingUpdater +Environment variables for the linkding worker. + +| Variable | Value | Description | Attention | +|------------------------|-----------|--------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Worker__Intervall | int (>=0) | This value sets the execution schedule in minutes. 1 = every minute, 10 = every 10 minutes | 0 = runs only one time. The container will be stopped after the execution. This method is the preferred way to run the container with a scheduler (e.g. cron) | +| Worker__SyncTag | text | The linkding tag to create the bookmarks in Wallabag. | | +| Linkding__Url | text | URL to the linkding instance | | +| Linkding__Key | text | The linkding application key | [Instructions](https://github.com/sissbruecker/linkding/blob/master/docs/API.md) | + +## Configuration +The following explains the configuration options. +### WallabagSync +The configuration is optional. In the configuration (**YAML File**) rules can be defined in regex to exclude certain domains from sync. + +Exampel: +````yaml +excludedDomains: +- name: youtube + pattern: 'https://[[a-zA-Z0-9]+\.]?(youtube)\.com(?:/.*)?' +- name: ebay + pattern: 'https://[[a-zA-Z0-9]+\.]?(ebay)\.(com|de|fr)(?:/.*)?' +- name: amazon + pattern: 'https://[[a-zA-Z0-9]+\.]?(amazon)\.(com|de|fr)(?:/.*)?' +```` +With this configuration every matching bookmark from linkding will be excluded from the sync. + +### LinkdingUpdater +The configuration is optional. In the configuration (**YAML File**) rules can be defined in regex to assign tags dynamically. Additionally tags can be defined to domains. + +If operated without a configuration file, only the year of the tag is added (currently). + +Example: +````yaml +urlTagMapping: + - name: microsoft_azure + url: https://github.com/azure + - name: microsoft_azuread + url: https://github.com/AzureAD + - name: microsoft_dotnet + url: https://github.com/dotnet-architecture + +taggingRule: + - name: reddit + pattern: https://(?:www\.)?(reddit)\.com(?:/r/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2 + - name: microsoft + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(microsoft)\.com(?:/.*)? + replace: $1,$2 + - name: microsoft_docs + pattern: 'https://(?:docs)\.(?:microsoft)\.com[ / ]?(?: [ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)?' + replace: $1,$2 + - name: youtube + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(youtube)\.com(?:/.*)? + replace: $1 + - name: ebay + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(ebay)\.(com|de|fr)(?:/.*)? + replace: $1 + - name: amazon + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(amazon)\.(com|de|fr)(?:/.*)? + replace: $1 + - name: docker + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(docker)\.com(?:/.*)? + replace: $1,$2 + - name: xbox + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(xbox)\.com(?:/.*)? + replace: $1 + - name: github + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(github)\.com[ / ]?([ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2,$3,$4 + - name: github.io + pattern: https://([ a-zA-Z0-9 ]+)\.(github)\.io[ / ]?([ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2,$3 +```` + +## Docker Run +``` +docker run --rm -it --env-file .env -v /config.yml:/app/data/config.yml linkdingsync/wallabag:latest +``` + +You can also + +## Docker Compose +You can find [examples](./examples/) in the examples folder.. + +- [Wallabag Example](./examples/wallabag/) +- [LinkdingUpdater Example](./examples/linkding/) + + +## Build Docker Image + +### LinkdingUpdater +``` +docker build -t linkdingsync/linkdingupdater:latest -f .\Dockerfile_Linkding . +``` + +### WallabagSync +``` +docker build -t linkdingsync/wallabag:latest -f .\Dockerfile_Wallabag . +``` \ No newline at end of file diff --git a/examples/linkding/config.yml b/examples/linkding/config.yml new file mode 100644 index 0000000..5e6e30a --- /dev/null +++ b/examples/linkding/config.yml @@ -0,0 +1,39 @@ +urlTagMapping: + - name: microsoft_azure + url: https://github.com/azure + - name: microsoft_azuread + url: https://github.com/AzureAD + - name: microsoft_dotnet + url: https://github.com/dotnet-architecture + +taggingRule: + - name: reddit + pattern: https://(?:www\.)?(reddit)\.com(?:/r/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2 + - name: microsoft + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(microsoft)\.com(?:/.*)? + replace: $1,$2 + - name: microsoft_docs + pattern: 'https://(?:docs)\.(?:microsoft)\.com[ / ]?(?: [ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)?' + replace: $1,$2 + - name: youtube + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(youtube)\.com(?:/.*)? + replace: $1 + - name: ebay + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(ebay)\.(com|de|fr)(?:/.*)? + replace: $1 + - name: amazon + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(amazon)\.(com|de|fr)(?:/.*)? + replace: $1 + - name: docker + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(docker)\.com(?:/.*)? + replace: $1,$2 + - name: xbox + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(xbox)\.com(?:/.*)? + replace: $1 + - name: github + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(github)\.com[ / ]?([ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2,$3,$4 + - name: github.io + pattern: https://([ a-zA-Z0-9 ]+)\.(github)\.io[ / ]?([ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2,$3 \ No newline at end of file diff --git a/examples/linkding/docker-compose.yml b/examples/linkding/docker-compose.yml new file mode 100644 index 0000000..23070c9 --- /dev/null +++ b/examples/linkding/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.9' + +services: + linkdingupdater: + image: linkdingsync/linkdingupdater:latest + volumes: + - ./config.yml:/data/config.yml + # env_file: + # - .env + environment: + - Worker__Intervall=0 + - Linkding__Url=https:// + - Linkding__Key= diff --git a/examples/wallabag/config/config.yml b/examples/wallabag/config/config.yml new file mode 100644 index 0000000..8283100 --- /dev/null +++ b/examples/wallabag/config/config.yml @@ -0,0 +1,7 @@ +excludedDomains: +- name: youtube + pattern: 'https://[[a-zA-Z0-9]+\.]?(youtube)\.com(?:/.*)?' +- name: ebay + pattern: 'https://[[a-zA-Z0-9]+\.]?(ebay)\.(com|de|fr)(?:/.*)?' +- name: amazon + pattern: 'https://[[a-zA-Z0-9]+\.]?(amazon)\.(com|de|fr)(?:/.*)?' \ No newline at end of file diff --git a/examples/wallabag/docker-compose.yml b/examples/wallabag/docker-compose.yml new file mode 100644 index 0000000..10ce0b9 --- /dev/null +++ b/examples/wallabag/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3.9' + +services: + wallabagsync: + image: linkdingsync/wallabag:latest + volumes: + - ./config.yml:/data/config.yml + # env_file: + # - .env + environment: + - Worker__Intervall=0 + - Worker__SyncTag= + - Linkding__Url=https:// + - Linkding__Key= + - Wallabag__Url=https:// + - Wallabag__Username= + - Wallabag__Password= + - Wallabag__ClientId= + - Wallabag__ClientSecret= diff --git a/global.json b/global.json new file mode 100644 index 0000000..1bcf6c0 --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "6.0.0", + "rollForward": "latestMinor", + "allowPrerelease": false + } +} \ No newline at end of file diff --git a/src/Domain/Core/Abstraction/ILinkdingService.cs b/src/Domain/Core/Abstraction/ILinkdingService.cs new file mode 100644 index 0000000..c3fc637 --- /dev/null +++ b/src/Domain/Core/Abstraction/ILinkdingService.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Core.Entities.Linkding; + +namespace Linkding.Client +{ + public interface ILinkdingService + { + Task> GetBookmarksAsync(int limit = 100, int offset = 0); + Task> GetAllBookmarksAsync(); + Task UpdateBookmarkCollectionAsync(IEnumerable bookmarks); + Task UpdateBookmarkCollectionAsync(IEnumerable bookmarks); + Task UpdateBookmarkAsync(BookmarkUpdatePayload bookmark); + Task GetBookmarkResultsAsync(int limit = 100, int offset = 0); + Task GetBookmarkResultsAsync(string url); + } +} \ No newline at end of file diff --git a/src/Domain/Core/Abstraction/IWallabagService.cs b/src/Domain/Core/Abstraction/IWallabagService.cs new file mode 100644 index 0000000..5e0a137 --- /dev/null +++ b/src/Domain/Core/Abstraction/IWallabagService.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading.Tasks; +using Core.Entities.Wallabag; + +namespace Core.Abstraction +{ + public interface IWallabagService + { + Task GetAuthenticationHeaderAsync(IEnumerable scopes = null); + + Task GetAsync(string endpoint, IEnumerable scopes = null, + bool httpCompletionResponseContentRead = false); + + Task GetJsonAsync(string endpoint, IEnumerable scopes = null); + + Task PostWithFormDataAsnyc(string endpoint, FormUrlEncodedContent content, + IEnumerable scopes = null); + + Task PostAsync(string endpoint, HttpContent content, + IEnumerable scopes = null); + + Task PostAsync(string endpoint, HttpRequestMessage request, + IEnumerable scopes = null); + + Task PutAsync(string endpoint, HttpContent content, + IEnumerable scopes = null); + + Task DeleteAsync(string endpoint, IEnumerable scopes = null, + HttpContent content = null); + + Task> GetEntries(string format = "json", int limit = 50, bool full = false); + Task GetEntryById(int id, string format = "json"); + Task AddEntryByUrl(string url, IEnumerable tags = null, string format = "json"); + } +} + diff --git a/src/Domain/Core/Converters/DateTimeConverterForCustomStandard.cs b/src/Domain/Core/Converters/DateTimeConverterForCustomStandard.cs new file mode 100644 index 0000000..983379b --- /dev/null +++ b/src/Domain/Core/Converters/DateTimeConverterForCustomStandard.cs @@ -0,0 +1,30 @@ +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Wallabag.Client.Converters +{ + public class DateTimeConverterForCustomStandard : JsonConverter + { + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateTimeString = reader.GetString(); + + if (string.IsNullOrEmpty(dateTimeString)) + { + return DateTime.MinValue; + } + DateTime dt = DateTime.ParseExact(dateTimeString, "yyyy-MM-dd'T'HH:mm:ssK", + CultureInfo.InvariantCulture, + DateTimeStyles.AdjustToUniversal); + + return dt; + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString()); + } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Converters/DateTimeOffsetConverterUsingDateTimeParse.cs b/src/Domain/Core/Converters/DateTimeOffsetConverterUsingDateTimeParse.cs new file mode 100644 index 0000000..a7d89a3 --- /dev/null +++ b/src/Domain/Core/Converters/DateTimeOffsetConverterUsingDateTimeParse.cs @@ -0,0 +1,20 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Wallabag.Client.Converters +{ + public class DateTimeOffsetConverterUsingDateTimeParse : JsonConverter + { + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateTimeString = reader.GetString(); + return DateTimeOffset.Parse(dateTimeString); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString()); + } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Core.csproj b/src/Domain/Core/Core.csproj new file mode 100644 index 0000000..7f2e911 --- /dev/null +++ b/src/Domain/Core/Core.csproj @@ -0,0 +1,14 @@ + + + + netstandard2.1 + enable + + + + + + + + + diff --git a/src/Domain/Core/Entities/Linkding/Bookmark.cs b/src/Domain/Core/Entities/Linkding/Bookmark.cs new file mode 100644 index 0000000..be6600a --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/Bookmark.cs @@ -0,0 +1,39 @@ +using System; +using System.Text.Json.Serialization; + +namespace Core.Entities.Linkding +{ + public class Bookmark : BookmarkBase + { + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("website_title")] + public string WebsiteTitle { get; set; } + [JsonPropertyName("website_description")] + public string WebsiteDescription { get; set; } + [JsonPropertyName("is_archived")] + public bool IsArchived { get; set; } + [JsonPropertyName("unread")] + public bool Unread { get; set; } + [JsonPropertyName("date_added")] + public DateTime DateAdded { get; set; } + [JsonPropertyName("date_modified")] + public DateTime DateModified { get; set; } + } +} + +// "id": 1, +// "url": "https://example.com", +// "title": "Example title", +// "description": "Example description", +// "website_title": "Website title", +// "website_description": "Website description", +// "is_archived": false, +// "unread": false, +// "tag_names": [ +// "tag1", +// "tag2" +// ], +// "date_added": "2020-09-26T09:46:23.006313Z", +// "date_modified": "2020-09-26T16:01:14.275335Z" \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/BookmarkBase.cs b/src/Domain/Core/Entities/Linkding/BookmarkBase.cs new file mode 100644 index 0000000..eb2e1f5 --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/BookmarkBase.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Core.Entities.Linkding +{ + public class BookmarkBase + { + [JsonPropertyName("url")] + public string Url { get; set; } + [JsonPropertyName("title")] + public string Title { get; set; } + [JsonPropertyName("description")] + public string Description { get; set; } + [JsonPropertyName("tag_names")] + public IEnumerable TagNames { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/BookmarkCreatePayload.cs b/src/Domain/Core/Entities/Linkding/BookmarkCreatePayload.cs new file mode 100644 index 0000000..c9b5361 --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/BookmarkCreatePayload.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Linkding +{ + public class BookmarkCreatePayload : BookmarkBase + { + [JsonPropertyName("is_archived")] + public bool IsArchived { get; set; } = false; + + [JsonPropertyName("unread")] public bool Unread { get; set; } = false; + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/BookmarkUpdatePayload.cs b/src/Domain/Core/Entities/Linkding/BookmarkUpdatePayload.cs new file mode 100644 index 0000000..b96cd85 --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/BookmarkUpdatePayload.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Linkding +{ + public class BookmarkUpdatePayload : BookmarkBase + { + [JsonPropertyName("id")] + public int Id { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/BookmarkUpdateResult.cs b/src/Domain/Core/Entities/Linkding/BookmarkUpdateResult.cs new file mode 100644 index 0000000..626030e --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/BookmarkUpdateResult.cs @@ -0,0 +1,7 @@ +namespace Core.Entities.Linkding +{ + public class BookmarkUpdateResult : BookmarkUpdatePayload + { + public bool Success { get; set; } = true; + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/BookmarksResult.cs b/src/Domain/Core/Entities/Linkding/BookmarksResult.cs new file mode 100644 index 0000000..17120e2 --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/BookmarksResult.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; + +namespace Core.Entities.Linkding +{ + public class BookmarksResult + { + public long Count { get; set; } + public string? Next { get; set; } + public string? Previous { get; set; } + public List Results { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/HandlerResult.cs b/src/Domain/Core/Entities/Linkding/HandlerResult.cs new file mode 100644 index 0000000..6ca539f --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/HandlerResult.cs @@ -0,0 +1,20 @@ +using Core.Entities.Linkding; + +namespace LinkdingUpdater.Handler +{ + public class HandlerResult + { + public bool PerformAction { get; set; } = false; + + public LinkdingItemAction Action { get; set; } + public bool HasError { get; set; } = false; + public string ErrorMessage { get; set; } = string.Empty; + public Bookmark Instance { get; set; } + } +} + +public enum LinkdingItemAction +{ + Update, + Delete +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/Tag.cs b/src/Domain/Core/Entities/Linkding/Tag.cs new file mode 100644 index 0000000..491b122 --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/Tag.cs @@ -0,0 +1,15 @@ +using System; +using System.Text.Json.Serialization; + +namespace Core.Entities.Linkding +{ + public class Tag + { + [JsonPropertyName("id")] + public int Id { get; set; } + [JsonPropertyName("name")] + public string Name { get; set; } + [JsonPropertyName("date_added")] + public DateTime DateAdded { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Linkding/TagCreatePayload.cs b/src/Domain/Core/Entities/Linkding/TagCreatePayload.cs new file mode 100644 index 0000000..0ed9c1f --- /dev/null +++ b/src/Domain/Core/Entities/Linkding/TagCreatePayload.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Linkding +{ + public class TagCreatePayload + { + [JsonPropertyName("name")] + public string Name { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/Annotation.cs b/src/Domain/Core/Entities/Wallabag/Annotation.cs new file mode 100644 index 0000000..05142ba --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/Annotation.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Wallabag.Client.Converters; + +namespace Core.Entities.Wallabag +{ + public class Annotation + { + [JsonPropertyName("user")] public string User { get; set; } + + [JsonPropertyName("annotator_schema_version")] + public string AnnotatorSchemaVersion { get; set; } + + [JsonPropertyName("id")] public int Id { get; set; } + + [JsonPropertyName("text")] public string Text { get; set; } + + + [JsonConverter(typeof(DateTimeConverterForCustomStandard))] + [JsonPropertyName("created_at")] public DateTime? CreatedAt { get; set; } = null; + + [JsonConverter(typeof(DateTimeConverterForCustomStandard))] + [JsonPropertyName("updated_at")] public DateTime? UpdatedAt { get; set; } = null; + + [JsonPropertyName("quote")] public string Quote { get; set; } + + [JsonPropertyName("ranges")] public List Ranges { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/Embedded.cs b/src/Domain/Core/Entities/Wallabag/Embedded.cs new file mode 100644 index 0000000..819c909 --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/Embedded.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class Embedded + { + [JsonPropertyName("items")] + public List Items { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/Headers.cs b/src/Domain/Core/Entities/Wallabag/Headers.cs new file mode 100644 index 0000000..a6d881f --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/Headers.cs @@ -0,0 +1,28 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class Headers + { + [JsonPropertyName("server")] public string Server { get; set; } + + [JsonPropertyName("date")] public string Date { get; set; } + + [JsonPropertyName("content-type")] public string ContentType { get; set; } + + [JsonPropertyName("content-length")] public string ContentLength { get; set; } + + [JsonPropertyName("connection")] public string Connection { get; set; } + + [JsonPropertyName("x-powered-by")] public string XPoweredBy { get; set; } + + [JsonPropertyName("cache-control")] public string CacheControl { get; set; } + + [JsonPropertyName("etag")] public string Etag { get; set; } + + [JsonPropertyName("vary")] public string Vary { get; set; } + + [JsonPropertyName("strict-transport-security")] + public string StrictTransportSecurity { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/HttpLink.cs b/src/Domain/Core/Entities/Wallabag/HttpLink.cs new file mode 100644 index 0000000..d274d9f --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/HttpLink.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class HttpLink + { + [JsonPropertyName("href")] public string Href { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/Links.cs b/src/Domain/Core/Entities/Wallabag/Links.cs new file mode 100644 index 0000000..741e6c4 --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/Links.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class Links + { + [JsonPropertyName("self")] public HttpLink? Self { get; set; } = null; + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/QueryLinks.cs b/src/Domain/Core/Entities/Wallabag/QueryLinks.cs new file mode 100644 index 0000000..59f43a5 --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/QueryLinks.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class QueryLinks : Links + { + [JsonPropertyName("first")] public HttpLink? First { get; set; } = null; + + [JsonPropertyName("last")] public HttpLink? Last { get; set; } = null; + + [JsonPropertyName("next")] public HttpLink? Next { get; set; } = null; + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/Range.cs b/src/Domain/Core/Entities/Wallabag/Range.cs new file mode 100644 index 0000000..401a85b --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/Range.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class Range + { + [JsonPropertyName("start")] public string Start { get; set; } + + [JsonPropertyName("startOffset")] public string StartOffset { get; set; } + + [JsonPropertyName("end")] public string End { get; set; } + + [JsonPropertyName("endOffset")] public string EndOffset { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/Tag.cs b/src/Domain/Core/Entities/Wallabag/Tag.cs new file mode 100644 index 0000000..f6d849b --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/Tag.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class Tag + { + [JsonPropertyName("id")] public int Id { get; set; } + + [JsonPropertyName("label")] public string Label { get; set; } + + [JsonPropertyName("slug")] public string Slug { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/WallabagItem.cs b/src/Domain/Core/Entities/Wallabag/WallabagItem.cs new file mode 100644 index 0000000..b9db188 --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/WallabagItem.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Wallabag.Client.Converters; + +namespace Core.Entities.Wallabag +{ + public class WallabagItem +{ + [JsonPropertyName("is_archived")] public int IsArchived { get; set; } + + [JsonPropertyName("is_starred")] public int IsStarred { get; set; } + + [JsonPropertyName("user_name")] public string UserName { get; set; } + + [JsonPropertyName("user_email")] public string UserEmail { get; set; } + + [JsonPropertyName("user_id")] public int UserId { get; set; } + + [JsonPropertyName("tags")] public List Tags { get; set; } + + [JsonPropertyName("is_public")] public bool IsPublic { get; set; } + + [JsonPropertyName("id")] public int Id { get; set; } + + [JsonPropertyName("uid")] public string? Uid { get; set; } = null; + + [JsonPropertyName("title")] public string Title { get; set; } + + [JsonPropertyName("url")] public string Url { get; set; } + + [JsonPropertyName("hashed_url")] public string HashedUrl { get; set; } + + [JsonPropertyName("origin_url")] public string? OriginUrl { get; set; } = null; + + [JsonPropertyName("given_url")] public string GivenUrl { get; set; } + + [JsonPropertyName("hashed_given_url")] public string HashedGivenUrl { get; set; } + + [JsonConverter(typeof(DateTimeConverterForCustomStandard))] + [JsonPropertyName("archived_at")] public DateTime? ArchivedAt { get; set; } = null; + + [JsonPropertyName("content")] public string? Content { get; set; } = null; + + [JsonConverter(typeof(DateTimeConverterForCustomStandard))] + [JsonPropertyName("created_at")] public DateTime CreatedAt { get; set; } + + [JsonConverter(typeof(DateTimeConverterForCustomStandard))] + [JsonPropertyName("updated_at")] public DateTime UpdatedAt { get; set; } + + [JsonConverter(typeof(DateTimeConverterForCustomStandard))] + [JsonPropertyName("published_at")] public DateTime PublishedAt { get; set; } + + [JsonPropertyName("published_by")] public List PublishedBy { get; set; } + + [JsonConverter(typeof(DateTimeConverterForCustomStandard))] + [JsonPropertyName("starred_at")] public DateTime? StarredAt { get; set; } = null; + + [JsonPropertyName("annotations")] public List Annotations { get; set; } + + [JsonPropertyName("mimetype")] public string Mimetype { get; set; } + + [JsonPropertyName("language")] public string Language { get; set; } + + [JsonPropertyName("reading_time")] public int ReadingTime { get; set; } + + [JsonPropertyName("domain_name")] public string DomainName { get; set; } + + [JsonPropertyName("preview_picture")] public string PreviewPicture { get; set; } + + [JsonPropertyName("http_status")] public string HttpStatus { get; set; } + + [JsonPropertyName("headers")] public Headers Headers { get; set; } + + [JsonPropertyName("_links")] public Links Links { get; set; } +} +} \ No newline at end of file diff --git a/src/Domain/Core/Entities/Wallabag/WallabagQuery.cs b/src/Domain/Core/Entities/Wallabag/WallabagQuery.cs new file mode 100644 index 0000000..261c5c8 --- /dev/null +++ b/src/Domain/Core/Entities/Wallabag/WallabagQuery.cs @@ -0,0 +1,25 @@ +using System.Text.Json.Serialization; + +namespace Core.Entities.Wallabag +{ + public class WallabagQuery + { + [JsonPropertyName("page")] + public int Page { get; set; } + + [JsonPropertyName("limit")] + public int Limit { get; set; } + + [JsonPropertyName("pages")] + public int Pages { get; set; } + + [JsonPropertyName("total")] + public int Total { get; set; } + + [JsonPropertyName("_links")] + public QueryLinks QueryLinks { get; set; } + + [JsonPropertyName("_embedded")] + public Embedded Embedded { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Core/Handler/ILinkdingTaskHandler.cs b/src/Domain/Core/Handler/ILinkdingTaskHandler.cs new file mode 100644 index 0000000..2f26959 --- /dev/null +++ b/src/Domain/Core/Handler/ILinkdingTaskHandler.cs @@ -0,0 +1,14 @@ +using System.Threading.Tasks; +using Core.Entities.Linkding; +using LinkdingUpdater.Handler; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Core.Handler +{ + public interface ILinkdingTaskHandler + { + string Command { get; } + Task ProcessAsync(Bookmark bookmark, ILogger logger, IConfiguration configuration); + } +} \ No newline at end of file diff --git a/src/Domain/Core/Handler/ISyncTaskHandler.cs b/src/Domain/Core/Handler/ISyncTaskHandler.cs new file mode 100644 index 0000000..987d385 --- /dev/null +++ b/src/Domain/Core/Handler/ISyncTaskHandler.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Core.Entities.Wallabag; +using Linkding.Client; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Core.Handler +{ + public interface ISyncTaskHandler + { + Type HandlerType { get; } + string Command { get; } + + Task ProcessAsync(IEnumerable items, T destinationService, ILinkdingService linkdingService, ILogger logger, IConfiguration configuration); + } +} \ No newline at end of file diff --git a/src/Linkding/Dockerfile b/src/Linkding/Dockerfile new file mode 100644 index 0000000..8d3775a --- /dev/null +++ b/src/Linkding/Dockerfile @@ -0,0 +1,19 @@ +FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build +WORKDIR /src +COPY ["src/LinkdingService/LinkdingService.csproj", "src/LinkdingService/"] +RUN dotnet restore "src/LinkdingService/LinkdingService.csproj" +COPY . . +WORKDIR "/src/src/LinkdingService" +RUN dotnet build "LinkdingService.csproj" -c Release -o /app/build + +FROM build AS publish +RUN dotnet publish "LinkdingService.csproj" -c Release -o /app/publish + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +RUN mkdir ./data +ENTRYPOINT ["dotnet", "LinkdingService.dll"] diff --git a/src/Linkding/Extensions/ServiceRegistrationExtensions.cs b/src/Linkding/Extensions/ServiceRegistrationExtensions.cs new file mode 100644 index 0000000..a911376 --- /dev/null +++ b/src/Linkding/Extensions/ServiceRegistrationExtensions.cs @@ -0,0 +1,18 @@ +using Linkding.Options; +using Linkding.Settings; + +// ReSharper disable once CheckNamespace +namespace Microsoft.Extensions.DependencyInjection; + +public static class ServiceRegistrationExtensions +{ + public static IServiceCollection Add_Linkding_Worker(this IServiceCollection services, + IConfiguration configuration) + { + var configSection = configuration.GetSection(WorkerSettings.Position); + services.Configure(configSection); + services.AddSingleton(); + + return services; + } +} \ No newline at end of file diff --git a/src/Linkding/Handler/AddPopularSitesAsTagHandler.cs b/src/Linkding/Handler/AddPopularSitesAsTagHandler.cs new file mode 100644 index 0000000..b606486 --- /dev/null +++ b/src/Linkding/Handler/AddPopularSitesAsTagHandler.cs @@ -0,0 +1,71 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using Core.Abstraction; +using Core.Entities.Linkding; +using Core.Handler; +using Linkding.Settings; +using LinkdingUpdater.Handler; + +namespace Linkding.Handler; + +public class AddPopularSitesAsTagHandler : ILinkdingTaskHandler +{ + private record RegexExpressionGroups(string Expression, string Replace); + + public string Command { get; } = "AddPopularSitesAsTag"; + public async Task ProcessAsync(Bookmark bookmark, ILogger logger, IConfiguration configuration) + { + var settings = SettingsService.Settings; + + var returnValue = new HandlerResult() {Instance = bookmark}; + Regex r = null; + Match m = null; + foreach (var regexEntry in settings.taggingRule) + { + try + { + r = new Regex(regexEntry.pattern, RegexOptions.IgnoreCase); + m = r.Match(returnValue.Instance.Url); + if (m.Success) + { + var tagsCommaSeparated = r.Replace(returnValue.Instance.Url, regexEntry.replace); + if (!string.IsNullOrEmpty(tagsCommaSeparated)) + { + var tags = tagsCommaSeparated.Split(','); + foreach (var tag in tags) + { + if (!string.IsNullOrEmpty(tag) && !returnValue.Instance.TagNames.Contains(tag) && + returnValue.Instance.TagNames.FirstOrDefault(x => x.ToLower() == tag.ToLower()) == null) + { + + returnValue.Instance.TagNames = returnValue.Instance.TagNames.Add(tag); + returnValue.PerformAction = true; + returnValue.Action = LinkdingItemAction.Update; + } + } + } + } + } + finally + { + r = null; + m = null; + } + + } + + + foreach (var urlKeyValue in settings.urlTagMapping) + { + if (returnValue.Instance.Url.ToLower().StartsWith(urlKeyValue.url.ToLower()) && returnValue.Instance.TagNames.FirstOrDefault(x => x.ToLower() == urlKeyValue.name.ToLower()) == null) + { + returnValue.Instance.TagNames = returnValue.Instance.TagNames.Add(urlKeyValue.name); + + returnValue.PerformAction = true; + } + } + + + return returnValue; + } +} diff --git a/src/Linkding/Handler/AddYearToBookmarkHandler.cs b/src/Linkding/Handler/AddYearToBookmarkHandler.cs new file mode 100644 index 0000000..1f502cd --- /dev/null +++ b/src/Linkding/Handler/AddYearToBookmarkHandler.cs @@ -0,0 +1,51 @@ +using Core.Abstraction; +using Core.Entities.Linkding; +using Core.Handler; +using LinkdingUpdater.Handler; + +namespace Linkding.Handler; + +public class AddYearToBookmarkHandler : ILinkdingTaskHandler +{ + public string Command { get; } = "AddYearToBookmark"; + + public async Task ProcessAsync(Bookmark bookmark, ILogger logger, IConfiguration configuration) + { + var returnValue = new HandlerResult() {Instance = bookmark}; + + var update = false; + var createdYear = returnValue.Instance.DateAdded.GetYear(); + + if (createdYear != "1970") + { + var tagName = returnValue.Instance.TagNames.FirstOrDefault(x => x.Equals(createdYear)); + if (tagName == null) + { + logger.LogInformation( + $"Detected bookmark ({returnValue.Instance.WebsiteTitle} - {returnValue.Instance.Id}) without year-tag ... Try to update"); + returnValue.Instance.TagNames = returnValue.Instance.TagNames.Add(createdYear); + update = true; + } + } + else + { + var wrongTagName = returnValue.Instance.TagNames.FirstOrDefault(x => x.Equals("1970")); + if (wrongTagName != null) + { + logger.LogInformation( + $"Detected bookmark ({returnValue.Instance.WebsiteTitle} - {returnValue.Instance.Id}) with '1970' year-tag ... Try to update"); + returnValue.Instance.TagNames = returnValue.Instance.TagNames.Where(x => !x.Equals("1970")).Select(x => x); + update = true; + } + } + + if (update) + { + logger.LogInformation($"Start updating bookmark {returnValue.Instance.WebsiteTitle} - {returnValue.Instance.Id}"); + returnValue.PerformAction = true; + returnValue.Action = LinkdingItemAction.Update; + } + + return returnValue; + } +} \ No newline at end of file diff --git a/src/Linkding/Linkding.csproj b/src/Linkding/Linkding.csproj new file mode 100644 index 0000000..01ed851 --- /dev/null +++ b/src/Linkding/Linkding.csproj @@ -0,0 +1,25 @@ + + + + net6.0 + enable + enable + dotnet-LinkdingService-80165DE2-FA70-4803-B366-DF8F24CF86BE + Linux + + + + + + + + + + .dockerignore + + + + + + + diff --git a/src/Linkding/Options/WorkerSettings.cs b/src/Linkding/Options/WorkerSettings.cs new file mode 100644 index 0000000..7b87cfa --- /dev/null +++ b/src/Linkding/Options/WorkerSettings.cs @@ -0,0 +1,8 @@ +namespace Linkding.Options; + +public class WorkerSettings +{ + public const string Position = "Worker"; + + public int Intervall { get; set; } = 0; +} \ No newline at end of file diff --git a/src/Linkding/Program.cs b/src/Linkding/Program.cs new file mode 100644 index 0000000..71d8ee2 --- /dev/null +++ b/src/Linkding/Program.cs @@ -0,0 +1,11 @@ +using Linkding; + +IHost host = Host.CreateDefaultBuilder(args) + .ConfigureServices((ctx, services) => + { + services.Add_Linkding_HttpClient(ctx.Configuration); + services.AddHostedService(); + }) + .Build(); + +await host.RunAsync(); \ No newline at end of file diff --git a/src/Linkding/Properties/launchSettings.json b/src/Linkding/Properties/launchSettings.json new file mode 100644 index 0000000..b93c3bf --- /dev/null +++ b/src/Linkding/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "LinkdingService": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/Linkding/Settings/SettingsService.cs b/src/Linkding/Settings/SettingsService.cs new file mode 100644 index 0000000..9c3f94e --- /dev/null +++ b/src/Linkding/Settings/SettingsService.cs @@ -0,0 +1,68 @@ +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Linkding.Settings; + +public class SettingYaml +{ + public List urlTagMapping { get; set; } = new (); + public List taggingRule { get; set; } = new (); +} + +public class UrlTagMapping +{ + public string name { get; set; } + public string url { get; set; } +} + +public class TaggingRule +{ + public string name { get; set; } + public string pattern { get; set; } + public string replace { get; set; } +} + +public class SettingsService +{ + private const string fileName = "data/config.yml"; + + private static SettingYaml _settings = null; + + public static SettingYaml Settings + { + get + { + if (_settings == null) + { + Initialize(); + } + + return _settings; + } + private set + { + _settings = value; + } + } + + private static void Initialize() + { + var filePath = Path.Combine(Environment.CurrentDirectory, fileName); + var fileInfo = new FileInfo(filePath); + + if (fileInfo.Exists) + { + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) // see height_in_inches in sample yml + .Build(); + + var yml = File.ReadAllText(fileInfo.FullName); + + Settings = deserializer.Deserialize(yml); + } + else + { + Settings = new SettingYaml(); + } + } +} \ No newline at end of file diff --git a/src/Linkding/Worker.cs b/src/Linkding/Worker.cs new file mode 100644 index 0000000..f41a26a --- /dev/null +++ b/src/Linkding/Worker.cs @@ -0,0 +1,178 @@ +using Core.Abstraction; +using Core.Entities.Linkding; +using Core.Handler; +using Linkding.Client; +using Linkding.Client.Options; +using Linkding.Options; +using Microsoft.Extensions.Options; + +namespace Linkding; + +public class Worker : BackgroundService +{ + private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly ILogger _logger; + private readonly LinkdingService _linkdingService; + private readonly LinkdingSettings _linkdingSettings; + private readonly WorkerSettings _settings; + private readonly IConfiguration _configuration; + + public Worker(ILogger logger, LinkdingService linkdingService, + IOptions linkdingSettings, IOptions settings, IConfiguration configuration, IHostApplicationLifetime hostApplicationLifetime) + { + _logger = logger; + _linkdingService = linkdingService; + _configuration = configuration; + _hostApplicationLifetime = hostApplicationLifetime; + _settings = settings.Value; + _linkdingSettings = linkdingSettings.Value; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); + + await RunTaskHandler(); + int delay = _settings.Intervall * 60000; + + if (delay > 0) + { + _logger.LogInformation($"Worker paused for: {_settings.Intervall} minutes"); + + await Task.Delay(delay, stoppingToken); + } + else + { + _logger.LogInformation($"Intervall value is '0' --> stopping worker"); + _hostApplicationLifetime.StopApplication(); + } + } + } + + public async Task RunTaskHandler() + { + if (!string.IsNullOrEmpty(_linkdingSettings.Url) && _linkdingSettings.UpdateBookmarks) + { + _logger.LogInformation($"Starting updating bookmarks for {_linkdingSettings.Url}"); + + _logger.LogInformation("Collecting Handler"); + var handlers = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ILinkdingTaskHandler).IsAssignableFrom(p) && p.IsClass); + + var updatedBookmarksCount = 0; + var updateBookmarks = new List(); + var deleteBookmarks = new List(); + if (handlers != null && handlers.Count() > 0) + { + var linkdingBookmarks = await _linkdingService.GetAllBookmarksAsync(); + if (linkdingBookmarks.Count() > 0) + { + + _logger.LogInformation($"{linkdingBookmarks.Count()} bookmarks found in {_linkdingSettings.Url}"); + + foreach (var handler in handlers) + { + ILinkdingTaskHandler handlerInstance = null; + try + { + handlerInstance = (ILinkdingTaskHandler) Activator.CreateInstance(handler); + + foreach (var linkdingBookmark in linkdingBookmarks) + { + try + { + _logger.LogDebug($"Start executing {handlerInstance.Command}"); + // var updateBookmark = updateBookmarks.FirstOrDefault(x => x.Id == linkdingBookmark.Id); + var existingBookmarkIndexInt = + updateBookmarks.FindIndex(x => x.Id == linkdingBookmark.Id); + + var bookmarkInstance = existingBookmarkIndexInt != -1 + ? updateBookmarks[existingBookmarkIndexInt] + : linkdingBookmark; + + var result = await handlerInstance.ProcessAsync(bookmarkInstance, _logger, _configuration); + + if (result.HasError) + { + _logger.LogWarning(result.ErrorMessage, handlerInstance.Command); + } + else + { + if (result.PerformAction) + { + if (result.Action == LinkdingItemAction.Delete) + { + if (existingBookmarkIndexInt != -1) + { + updateBookmarks.RemoveAt(existingBookmarkIndexInt); + } + + var bookmarkToDelete = deleteBookmarks.FirstOrDefault(x => + x.Url.ToLower() == result.Instance.Url.ToLower()); + if (bookmarkToDelete == null) + { + deleteBookmarks.Add(result.Instance); + } + } + else + { + if (existingBookmarkIndexInt != -1) + { + updateBookmarks[existingBookmarkIndexInt] = result.Instance; + } + else + { + updateBookmarks.Add(result.Instance); + } + } + } + } + + _logger.LogDebug($"Finished {handlerInstance.Command}"); + } + catch (Exception e) + { + Console.WriteLine(e); + var message = $"... {e.Message}"; + + if (handlerInstance != null && !string.IsNullOrEmpty(handlerInstance.Command)) + { + message = $"Error while executing {handlerInstance.Command}! {message}"; + } + else + { + message = $"Error while executing handler! {message}"; + } + + _logger.LogError(message, "Calling Handler", e); + // throw; + } + } + } + catch (Exception e) + { + Console.WriteLine(e); + throw; + } + } + } + else + { + _logger.LogInformation($"no bookmarks found in {_linkdingSettings.Url}"); + } + + if (updateBookmarks.Count() > 0) + { + _logger.LogDebug($"Start updating bookmarks"); + await _linkdingService.UpdateBookmarkCollectionAsync(updateBookmarks); + _logger.LogDebug($"Successfully updated bookmarks"); + } + } + + _logger.LogInformation($"Finished updating bookmarks for {_linkdingSettings.Url}"); + } + } +} \ No newline at end of file diff --git a/src/Linkding/appsettings.Development.json b/src/Linkding/appsettings.Development.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/src/Linkding/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/Linkding/appsettings.json b/src/Linkding/appsettings.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/src/Linkding/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/Linkding/config.yml b/src/Linkding/config.yml new file mode 100644 index 0000000..fa05237 --- /dev/null +++ b/src/Linkding/config.yml @@ -0,0 +1,39 @@ +urlTagMapping: + - name: microsoft_azure + url: https://github.com/azure + - name: microsoft_azuread + url: https://github.com/AzureAD + - name: microsoft_dotnet + url: https://github.com/dotnet-architecture + +taggingRule: + - name: reddit + pattern: https://(?:www\.)?(reddit)\.com(?:/r/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2 + - name: microsoft + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(microsoft)\.com(?:/.*)? + replace: $1,$2 + - name: microsoft_docs + pattern: "https://(?:docs)\.(?:microsoft)\.com[ / ]?(?: [ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)?" + replace: $1,$2 + - name: youtube + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(youtube)\.com(?:/.*)? + replace: $1 + - name: ebay + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(ebay)\.(com|de|fr)(?:/.*)? + replace: $1 + - name: amazon + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(amazon)\.(com|de|fr)(?:/.*)? + replace: $1 + - name: docker + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(docker)\.com(?:/.*)? + replace: $1,$2 + - name: xbox + pattern: https://[ [ a-zA-Z0-9 ]+\. ]?(xbox)\.com(?:/.*)? + replace: $1 + - name: github + pattern: https://([ a-zA-Z0-9 ]+)?[ \. ]?(github)\.com[ / ]?([ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2,$3,$4 + - name: github.io + pattern: https://([ a-zA-Z0-9 ]+)\.(github)\.io[ / ]?([ a-zA-Z0-9\-\+_ ]+)(?:/)?([ a-zA-Z0-9\-\+_ ]+)?(?:/.*)? + replace: $1,$2,$3 \ No newline at end of file diff --git a/src/Services/Linkding.Client/Automapper/LinkdingBookmarkProfile.cs b/src/Services/Linkding.Client/Automapper/LinkdingBookmarkProfile.cs new file mode 100644 index 0000000..df39869 --- /dev/null +++ b/src/Services/Linkding.Client/Automapper/LinkdingBookmarkProfile.cs @@ -0,0 +1,12 @@ +using AutoMapper; +using Core.Entities.Linkding; + +namespace Linkding.Client.Automapper; + +public class LinkdingBookmarkProfile : Profile +{ + public LinkdingBookmarkProfile() + { + CreateMap(); + } +} \ No newline at end of file diff --git a/src/Services/Linkding.Client/Extensions/ServiceRegistrationExtensions.cs b/src/Services/Linkding.Client/Extensions/ServiceRegistrationExtensions.cs new file mode 100644 index 0000000..7858b51 --- /dev/null +++ b/src/Services/Linkding.Client/Extensions/ServiceRegistrationExtensions.cs @@ -0,0 +1,35 @@ +using System.Net.Http.Headers; +using Linkding.Client; +using Linkding.Client.Options; +using Microsoft.Extensions.Configuration; +using Polly; +using Polly.Extensions.Http; + +// ReSharper disable once CheckNamespace +namespace Microsoft.Extensions.DependencyInjection; + +public static class ServiceRegistrationExtensions +{ + public static IServiceCollection Add_Linkding_HttpClient(this IServiceCollection services, + IConfiguration configuration) + { + var configSection = configuration.GetSection(LinkdingSettings.Position); + services.Configure(configSection); + services.AddHttpClient() + .SetHandlerLifetime(TimeSpan.FromMinutes(5)) //Set lifetime to five minutes + .AddPolicyHandler(GetRetryPolicy()); + + services.AddAutoMapper(AppDomain.CurrentDomain.GetAssemblies()); + + return services; + } + + static IAsyncPolicy GetRetryPolicy() + { + return HttpPolicyExtensions + .HandleTransientHttpError() + .OrResult(msg => msg.StatusCode == System.Net.HttpStatusCode.NotFound) + .WaitAndRetryAsync(6, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, + retryAttempt))); + } +} \ No newline at end of file diff --git a/src/Services/Linkding.Client/Extensions/System.cs b/src/Services/Linkding.Client/Extensions/System.cs new file mode 100644 index 0000000..0d22ee3 --- /dev/null +++ b/src/Services/Linkding.Client/Extensions/System.cs @@ -0,0 +1,38 @@ +namespace System; + +public static class System +{ + public static DateTime CreateDateTime(this long ticks) + { + var dateTimeOffset = DateTimeOffset.FromUnixTimeSeconds(ticks); + + return dateTimeOffset.UtcDateTime; + } + + public static DateTime CreateDateTime(this string ticksString) + { + if (long.TryParse(ticksString, out var dateAddedUnixEpoch)) + { + return dateAddedUnixEpoch.CreateDateTime(); + } + + return default; + } + + public static string GetYear(this DateTime date) + { + return date.ToString("yyyy"); + } + + public static string GetYear(this long ticks) + { + var dateTime = ticks.CreateDateTime(); + return dateTime.GetYear(); + } + + public static string GetYear(this string ticksString) + { + var dateTime = ticksString.CreateDateTime(); + return dateTime.GetYear(); + } +} \ No newline at end of file diff --git a/src/Services/Linkding.Client/Extensions/SystemCollectionGenericExtesions.cs b/src/Services/Linkding.Client/Extensions/SystemCollectionGenericExtesions.cs new file mode 100644 index 0000000..e187c59 --- /dev/null +++ b/src/Services/Linkding.Client/Extensions/SystemCollectionGenericExtesions.cs @@ -0,0 +1,12 @@ +// ReSharper disable once CheckNamespace +namespace System.Collections.Generic; + +public static class SystemCollectionGenericExtesions +{ + public static IEnumerable Add(this IEnumerable e, T value) { + foreach ( var cur in e) { + yield return cur; + } + yield return value; + } +} \ No newline at end of file diff --git a/src/Services/Linkding.Client/Linkding.Client.csproj b/src/Services/Linkding.Client/Linkding.Client.csproj new file mode 100644 index 0000000..8a97f10 --- /dev/null +++ b/src/Services/Linkding.Client/Linkding.Client.csproj @@ -0,0 +1,34 @@ + + + + net6.0 + enable + enable + AnyCPU + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Services/Linkding.Client/LinkdingService.cs b/src/Services/Linkding.Client/LinkdingService.cs new file mode 100644 index 0000000..74e9265 --- /dev/null +++ b/src/Services/Linkding.Client/LinkdingService.cs @@ -0,0 +1,127 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using AutoMapper; +using Core.Entities.Linkding; +using Linkding.Client.Options; +using Microsoft.Extensions.Options; + +namespace Linkding.Client; + +public class LinkdingService : ILinkdingService +{ + private readonly LinkdingSettings _settings; + private readonly IMapper _mapper; + public readonly HttpClient _client; + + public LinkdingService(HttpClient client, IOptions settings, IMapper mapper) + { + _settings = settings.Value; + _client = client; + _mapper = mapper; + _client.BaseAddress = new Uri(_settings.Url); + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", _settings.Key); + } + + private LinkdingService(string url, string key) + { + _client = new HttpClient(); + _client.BaseAddress = new Uri(url); + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", key); + } + + public async Task> GetBookmarksAsync(int limit = 100, int offset = 0) + { + var bookmarks = new List(); + + var result = await GetBookmarkResultsAsync(limit, offset); + if (result != null && result.Results?.Count() > 0) + { + bookmarks = result.Results; + } + + return bookmarks; + } + + public async Task> GetAllBookmarksAsync() + { + IEnumerable bookmarks = new List(); + + var result = await GetBookmarkResultsAsync(); + if (result != null && result.Results?.Count() > 0) + { + bookmarks = result.Results; + if (result.Count > 100) + { + while (!string.IsNullOrEmpty(result.Next)) + { + result = await GetBookmarkResultsAsync(result.Next); + if (result.Results?.Count() > 0) + { + bookmarks = bookmarks.Concat(result.Results); + } + else + { + break; + } + } + } + } + + return bookmarks; + } + + public async Task UpdateBookmarkCollectionAsync(IEnumerable bookmarks) + { + foreach (var bookmark in bookmarks) + { + var payload = _mapper.Map(bookmark); + await UpdateBookmarkAsync(payload); + } + } + + public async Task UpdateBookmarkCollectionAsync(IEnumerable bookmarks) + { + foreach (var bookmark in bookmarks) + { + await UpdateBookmarkAsync(bookmark); + } + } + + public async Task UpdateBookmarkAsync(BookmarkUpdatePayload bookmark) + { + var result = await _client.PutAsJsonAsync($"/api/bookmarks/{bookmark.Id}/", bookmark); + if (result.IsSuccessStatusCode) + { + + } + else + { + + } + } + + public async Task GetBookmarkResultsAsync(int limit = 100, int offset = 0) + { + BookmarksResult bookmarkResult = null; + + var url = $"/api/bookmarks/"; + + bookmarkResult = await GetBookmarkResultsAsync(url); + + return bookmarkResult; + } + + public async Task GetBookmarkResultsAsync(string url) + { + BookmarksResult bookmarkResult = null; + + bookmarkResult = await _client.GetFromJsonAsync(url); + + return bookmarkResult; + } + + public static LinkdingService Create(string url, string key) + { + return new LinkdingService(url, key); + } +} \ No newline at end of file diff --git a/src/Services/Linkding.Client/Options/LinkdingSettings.cs b/src/Services/Linkding.Client/Options/LinkdingSettings.cs new file mode 100644 index 0000000..df899db --- /dev/null +++ b/src/Services/Linkding.Client/Options/LinkdingSettings.cs @@ -0,0 +1,10 @@ +namespace Linkding.Client.Options; + +public class LinkdingSettings +{ + public const string Position = "Linkding"; + + public string Key { get; set; } + public string Url { get; set; } + public bool UpdateBookmarks { get; set; } = true; +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Contracts/IAccessTokenProvider.cs b/src/Services/Wallabag.Client/Contracts/IAccessTokenProvider.cs new file mode 100644 index 0000000..3e75a7c --- /dev/null +++ b/src/Services/Wallabag.Client/Contracts/IAccessTokenProvider.cs @@ -0,0 +1,6 @@ +namespace Wallabag.Client.Contracts; + +public interface IAccessTokenProvider +{ + Task GetToken(IEnumerable scopes); +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Converters/DateTimeConverterForCustomStandard.cs b/src/Services/Wallabag.Client/Converters/DateTimeConverterForCustomStandard.cs new file mode 100644 index 0000000..7ba4f3a --- /dev/null +++ b/src/Services/Wallabag.Client/Converters/DateTimeConverterForCustomStandard.cs @@ -0,0 +1,28 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Wallabag.Client.Converters; + +public class DateTimeConverterForCustomStandard : JsonConverter +{ + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateTimeString = reader.GetString(); + + if (string.IsNullOrEmpty(dateTimeString)) + { + return DateTime.MinValue; + } + DateTime dt = DateTime.ParseExact(dateTimeString, "yyyy-MM-dd'T'HH:mm:ssK", + CultureInfo.InvariantCulture, + DateTimeStyles.AdjustToUniversal); + + return dt; + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString()); + } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Converters/DateTimeOffsetConverterUsingDateTimeParse.cs b/src/Services/Wallabag.Client/Converters/DateTimeOffsetConverterUsingDateTimeParse.cs new file mode 100644 index 0000000..a2899d2 --- /dev/null +++ b/src/Services/Wallabag.Client/Converters/DateTimeOffsetConverterUsingDateTimeParse.cs @@ -0,0 +1,18 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Wallabag.Client.Converters; + +public class DateTimeOffsetConverterUsingDateTimeParse : JsonConverter +{ + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateTimeString = reader.GetString(); + return DateTimeOffset.Parse(dateTimeString); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString()); + } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Extensions/ServiceRegistrationExtensions.cs b/src/Services/Wallabag.Client/Extensions/ServiceRegistrationExtensions.cs new file mode 100644 index 0000000..e6ac3ff --- /dev/null +++ b/src/Services/Wallabag.Client/Extensions/ServiceRegistrationExtensions.cs @@ -0,0 +1,39 @@ +using System.Net.Http.Headers; +using Microsoft.Extensions.Configuration; +using Polly; +using Polly.Extensions.Http; +using Wallabag.Client; +using Wallabag.Client.Contracts; +using Wallabag.Client.OAuth; +using Wallabag.Client.Options; + +// ReSharper disable once CheckNamespace +namespace Microsoft.Extensions.DependencyInjection; + +public static class ServiceRegistrationExtensions +{ + public static IServiceCollection Add_Wallabag_HttpClient(this IServiceCollection services, + IConfiguration configuration) + { + var configSection = configuration.GetSection(WallabagSettings.Position); + services.Configure(configSection); + services.AddScoped(); + services.AddScoped(); + services.AddHttpClient() + .SetHandlerLifetime(TimeSpan.FromMinutes(5)) //Set lifetime to five minutes + .AddPolicyHandler(GetRetryPolicy()); + + // services.AddAutoMapper(AppDomain.CurrentDomain.GetAssemblies()); + + return services; + } + + static IAsyncPolicy GetRetryPolicy() + { + return HttpPolicyExtensions + .HandleTransientHttpError() + .OrResult(msg => msg.StatusCode == System.Net.HttpStatusCode.NotFound) + .WaitAndRetryAsync(6, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, + retryAttempt))); + } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Extensions/URIExtensions.cs b/src/Services/Wallabag.Client/Extensions/URIExtensions.cs new file mode 100644 index 0000000..d014b67 --- /dev/null +++ b/src/Services/Wallabag.Client/Extensions/URIExtensions.cs @@ -0,0 +1,26 @@ +// ReSharper disable once CheckNamespace +namespace System; + +public static class URIExtensions +{ + public static string AppendToURL(this string uri1, string uri2) + { + return AppendToUrlInternal(uri1, uri2); + } + + public static string AppendToURL(this string baseURL, params string[] segments) + { + return AppendToUrlInternal(baseURL, segments); + } + + private static string AppendToUrlInternal(this string baseURL, params string[] segments) + { + return string.Join("/", new[] { baseURL.TrimEnd('/') } + .Concat(segments.Select(s => s.Trim('/')))); + } + + public static Uri Append(this Uri uri, params string[] paths) + { + return new Uri(paths.Aggregate(uri.AbsoluteUri, (current, path) => string.Format("{0}/{1}", current.TrimEnd('/'), path.TrimStart('/')))); + } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Models/WallabagEntry.cs b/src/Services/Wallabag.Client/Models/WallabagEntry.cs new file mode 100644 index 0000000..4acad48 --- /dev/null +++ b/src/Services/Wallabag.Client/Models/WallabagEntry.cs @@ -0,0 +1,6 @@ +namespace Wallabag.Client.Models; + +public class WallabagEntry : WallabagPayload +{ + public int Id { get; set; } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Models/WallabagPayload.cs b/src/Services/Wallabag.Client/Models/WallabagPayload.cs new file mode 100644 index 0000000..dfd1080 --- /dev/null +++ b/src/Services/Wallabag.Client/Models/WallabagPayload.cs @@ -0,0 +1,17 @@ +namespace Wallabag.Client.Models; + +public class WallabagPayload +{ + public string Url { get; set; } + public string Title { get; set; } + public string Tags { get; set; } + public int Archive { get; set; } + public int Starred { get; set; } + public string Content { get; set; } + public string Language { get; set; } + public string PreviewPicture { get; set; } + public DateTime PublishedAt { get; set; } + public string Authors { get; set; } + public int Publich { get; set; } + public string OriginUrl { get; set; } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/OAuth/AuthenticationClient.cs b/src/Services/Wallabag.Client/OAuth/AuthenticationClient.cs new file mode 100644 index 0000000..ea3d77c --- /dev/null +++ b/src/Services/Wallabag.Client/OAuth/AuthenticationClient.cs @@ -0,0 +1,77 @@ +using System.Net.Http.Json; +using Microsoft.Extensions.Options; +using Wallabag.Client.Options; + +namespace Wallabag.Client.OAuth; + +public record WallabagToken(string token_type, string access_token, string refresh_token, int expires_in, string scope); + +public class AuthenticationClient +{ + protected const string AuthPath = "/oauth/v2/token"; + + public readonly HttpClient _client; + public readonly WallabagSettings _settings; + + public AuthenticationClient(HttpClient client, IOptions settings) + { + _client = client; + _settings = settings.Value; + // var baseAdress = new Uri(_settings.Url); + // client.BaseAddress = baseAdress.Append(AuthPath); + client.BaseAddress = new Uri(_settings.Url); + } + + public async Task Authenticate(IEnumerable scopes = null) + { + var parameters = new List> + { + new("client_id", _settings.ClientId), + new("client_secret", _settings.ClientSecret), + new("grant_type", _settings.GrandType), + new("username", _settings.Username), + new("password", _settings.Password) + }; + + if (scopes != null && scopes.Count() > 0) + { + parameters.Add(new("scope", string.Join(" ", scopes))); + } + var response = await _client.PostAsync("oauth/v2/token", new FormUrlEncodedContent(parameters)); + + response.EnsureSuccessStatusCode(); + var authentication = await response.Content.ReadFromJsonAsync(); + if (authentication == null) + { + throw new HttpRequestException("Could not retrieve authentication data."); + } + return authentication; + } + public async Task RefreshToken(string refreshToken, IEnumerable scopes = null) + { + var parameters = new List> + { + new("client_id", _settings.ClientId), + new("client_secret", _settings.ClientSecret), + new("grant_type", "refresh_token"), + new("refresh_token", refreshToken), + new("username", _settings.Username), + new("password", _settings.Password) + }; + + if (scopes != null && scopes.Count() > 0) + { + parameters.Add(new("scope", string.Join(" ", scopes))); + } + + var response = await _client.PostAsync("oauth/v2/token", new FormUrlEncodedContent(parameters)); + + response.EnsureSuccessStatusCode(); + var authentication = await response.Content.ReadFromJsonAsync(); + if (authentication == null) + { + throw new HttpRequestException("Could not retrieve authentication data."); + } + return authentication; + } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/OAuth/OAuthTokenProvider.cs b/src/Services/Wallabag.Client/OAuth/OAuthTokenProvider.cs new file mode 100644 index 0000000..6bb2cf4 --- /dev/null +++ b/src/Services/Wallabag.Client/OAuth/OAuthTokenProvider.cs @@ -0,0 +1,43 @@ +using Wallabag.Client.Contracts; + +namespace Wallabag.Client.OAuth; + +public class OAuthTokenProvider : IAccessTokenProvider +{ + protected record TokenCache(string access_token, DateTime expires); + + private readonly AuthenticationClient _client; + + protected Dictionary cache = new (); + + public OAuthTokenProvider(AuthenticationClient client) + { + _client = client; + } + + public async Task GetToken(IEnumerable scopes = null) + { + string cacheKey = "wallabag+token"; + + if (scopes != null && scopes.Count() > 0) + { + cacheKey = string.Join('+', scopes); + } + + if (cache.ContainsKey(cacheKey)) + { + var tokenCache = cache[cacheKey]; + if (tokenCache.expires > DateTime.Now) + { + return tokenCache.access_token; + } + else + { + cache.Remove(cacheKey); + } + } + var auth = await _client.Authenticate(scopes); + cache.Add(cacheKey, new TokenCache(auth.access_token, DateTime.Now.AddSeconds(auth.expires_in))); + return auth.access_token; + } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Options/WallabagSettings.cs b/src/Services/Wallabag.Client/Options/WallabagSettings.cs new file mode 100644 index 0000000..9def70f --- /dev/null +++ b/src/Services/Wallabag.Client/Options/WallabagSettings.cs @@ -0,0 +1,13 @@ +namespace Wallabag.Client.Options; + +public class WallabagSettings +{ + public const string Position = "Wallabag"; + + public string Url { get; set; } = "https://app.wallabag.it"; + public string Username { get; set; } + public string Password { get; set; } + public string ClientId { get; set; } + public string ClientSecret { get; set; } + public string GrandType { get; set; } = "password"; +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/Wallabag.Client.csproj b/src/Services/Wallabag.Client/Wallabag.Client.csproj new file mode 100644 index 0000000..8684a5d --- /dev/null +++ b/src/Services/Wallabag.Client/Wallabag.Client.csproj @@ -0,0 +1,34 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Services/Wallabag.Client/WallabagServiceBase.cs b/src/Services/Wallabag.Client/WallabagServiceBase.cs new file mode 100644 index 0000000..fcdc078 --- /dev/null +++ b/src/Services/Wallabag.Client/WallabagServiceBase.cs @@ -0,0 +1,108 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using Core.Abstraction; +using Microsoft.Extensions.Options; +using Wallabag.Client.Contracts; +using Wallabag.Client.Converters; +using Wallabag.Client.Options; + +namespace Wallabag.Client; + +public partial class WallabagService : IWallabagService +{ + private readonly WallabagSettings _settings; + private IAccessTokenProvider _accessTokenProvider; + public readonly HttpClient _client; + + public WallabagService(HttpClient client, IOptions settings, + IAccessTokenProvider accessTokenProvider) + { + _client = client; + _accessTokenProvider = accessTokenProvider; + _settings = settings.Value; + _client.BaseAddress = new Uri(_settings.Url); + } + + public async Task GetAuthenticationHeaderAsync(IEnumerable scopes = null) + { + return new AuthenticationHeaderValue("Bearer", await _accessTokenProvider.GetToken(scopes)); + } + + public async Task GetAsync(string endpoint, IEnumerable scopes = null, + bool httpCompletionResponseContentRead = false) + { + var request = new HttpRequestMessage(HttpMethod.Get, endpoint); + request.Headers.Authorization = await GetAuthenticationHeaderAsync(scopes); + + HttpResponseMessage response = null; + + if (httpCompletionResponseContentRead) + { + response = await _client.SendAsync(request, HttpCompletionOption.ResponseContentRead); + } + else + { + response = await _client.SendAsync(request); + } + + response.EnsureSuccessStatusCode(); + return response; + } + + public async Task GetJsonAsync(string endpoint, IEnumerable scopes = null) + { + var response = await GetAsync(endpoint, scopes); + + return await response.Content.ReadFromJsonAsync(); + } + + public async Task PostWithFormDataAsnyc(string endpoint, FormUrlEncodedContent content, + IEnumerable scopes = null) + { + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint); + request.Content = content; + + return await PostAsync(endpoint, request, scopes); + } + + public async Task PostAsync(string endpoint, HttpContent content, + IEnumerable scopes = null) + { + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint); + request.Content = content; + + return await PostAsync(endpoint, request, scopes); + } + + public async Task PostAsync(string endpoint, HttpRequestMessage request, + IEnumerable scopes = null) + { + request.Headers.Authorization = await GetAuthenticationHeaderAsync(scopes); + var response = await _client.SendAsync(request); + response.EnsureSuccessStatusCode(); + return response; + } + + public async Task PutAsync(string endpoint, HttpContent content, + IEnumerable scopes = null) + { + using var request = new HttpRequestMessage(HttpMethod.Put, endpoint); + request.Headers.Authorization = await GetAuthenticationHeaderAsync(scopes); + request.Content = content; + var response = await _client.SendAsync(request); + response.EnsureSuccessStatusCode(); + return response; + } + + public async Task DeleteAsync(string endpoint, IEnumerable scopes = null, + HttpContent content = null) + { + using var request = new HttpRequestMessage(HttpMethod.Delete, endpoint); + request.Headers.Authorization = await GetAuthenticationHeaderAsync(scopes); + request.Content = content; + var response = await _client.SendAsync(request); + response.EnsureSuccessStatusCode(); + return response; + } +} \ No newline at end of file diff --git a/src/Services/Wallabag.Client/WallabagServiceEntries.cs b/src/Services/Wallabag.Client/WallabagServiceEntries.cs new file mode 100644 index 0000000..8e5b10a --- /dev/null +++ b/src/Services/Wallabag.Client/WallabagServiceEntries.cs @@ -0,0 +1,85 @@ +using System.Net.Http.Json; +using Core.Entities.Wallabag; +using Wallabag.Client.Models; + +namespace Wallabag.Client; + +public partial class WallabagService +{ + public async Task> GetEntries(string format = "json", int limit = 50, bool full = false) + { + var bookmarks = new List(); + var url = $"/api/entries.{format}?perPage={limit}"; + if (!full) + { + url = $"{url}&detail=metadata"; + } + + var allQuery = await GetJsonAsync(url); + + if (allQuery != null && allQuery.Embedded != null && allQuery.Embedded.Items != null && allQuery.Embedded.Items.Count() > 0) + { + bookmarks = allQuery.Embedded.Items; + + if (allQuery.Total > limit) + { + while (allQuery.QueryLinks.Next != null && !string.IsNullOrEmpty(allQuery.QueryLinks.Next.Href)) + { + // url = allQuery.QueryLinks.Next.Href.Replace(_settings.Url, ""); + url = allQuery.QueryLinks.Next.Href.Replace("http://", "https://"); + allQuery = await GetJsonAsync(url); + bookmarks.AddRange(allQuery.Embedded.Items); + } + } + } + + return bookmarks; + } + + public async Task GetEntryById(int id, string format = "json") + { + var url = $"/api/entries/{id}.{format}"; + var item = await GetJsonAsync(url); + + return item; + } + + public async Task AddEntryByUrl(string url, IEnumerable tags = null, string format = "json") + { + var endpoint = $"/api/entries.{format}"; + + var keyVals = new Dictionary(); + keyVals.Add("url", url); + if (tags != null && tags.Count() > 0) + { + keyVals.Add("tags", string.Join(",", tags)); + } + + var content = new FormUrlEncodedContent(keyVals); + var response = await PostWithFormDataAsnyc(endpoint, content); + + var item = await response.Content.ReadFromJsonAsync(); + return item; + } + + // private async Task GetBookmarkResultsAsync(int limit = 100, int offset = 0) + // { + // WallabagEntry bookmarkResult = null; + // + // var url = $"/api/bookmarks/"; + // + // bookmarkResult = await GetBookmarkResultsAsync(url); + // + // return bookmarkResult; + // } + // + // private async Task GetBookmarkResultsAsync(string url) + // { + // WallabagEntry bookmarkResult = null; + // + // bookmarkResult = await _client.GetFromJsonAsync(url); + // + // return bookmarkResult; + // } + +} \ No newline at end of file diff --git a/src/Wallabag/Extensions/ServiceRegistrationExtensions.cs b/src/Wallabag/Extensions/ServiceRegistrationExtensions.cs new file mode 100644 index 0000000..b206da7 --- /dev/null +++ b/src/Wallabag/Extensions/ServiceRegistrationExtensions.cs @@ -0,0 +1,18 @@ +using Wallabag.Options; +using Wallabag.Settings; + +// ReSharper disable once CheckNamespace +namespace Microsoft.Extensions.DependencyInjection; + +public static class ServiceRegistrationExtensions +{ + public static IServiceCollection Add_Wallabag_Worker(this IServiceCollection services, + IConfiguration configuration) + { + var configSection = configuration.GetSection(WorkerSettings.Position); + services.Configure(configSection); + services.AddSingleton(); + + return services; + } +} \ No newline at end of file diff --git a/src/Wallabag/Handler/LinkdingBookmarkToWallabagHandler.cs b/src/Wallabag/Handler/LinkdingBookmarkToWallabagHandler.cs new file mode 100644 index 0000000..4c26fd0 --- /dev/null +++ b/src/Wallabag/Handler/LinkdingBookmarkToWallabagHandler.cs @@ -0,0 +1,91 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using Core.Entities.Wallabag; +using Core.Handler; +using Linkding.Client; +using Wallabag.Client; +using Wallabag.Settings; + +namespace Wallabag.Handler +{ + public class LinkdingBookmarkToWallabagHandler : ISyncTaskHandler + { + public Type HandlerType { get; } = typeof(WallabagService); + public string Command { get; } = "LinkdingBookmarkToWallabag"; + + public async Task ProcessAsync(IEnumerable items, WallabagService destinationService, + ILinkdingService linkdingService, + ILogger logger, IConfiguration configuration) + { + var wallabagsNormalized = new Dictionary(); + var updatedWallabags = new Dictionary>(); + var wallabagToRemove = new List(); + var linkdingBookmarks = await linkdingService.GetAllBookmarksAsync(); + + if (linkdingBookmarks != null && linkdingBookmarks.Count() > 0) + { + var settings = SettingsService.Settings; + + var tagName = configuration.GetValue("Worker:SyncTag"); + + linkdingBookmarks = + linkdingBookmarks.Where(x => x.TagNames.Contains(tagName)).OrderBy(x => x.DateAdded); + + Regex r = null; + Match m = null; + foreach (var bookmark in linkdingBookmarks) + { + var cleanUrl = + bookmark.Url.Replace( + "?utm_source=share&utm_medium=android_app&utm_name=androidcss&utm_term=2&utm_content=share_button", + ""); + // var existingElement = items.FirstOrDefault(x => x.Url.ToLower() == cleanUrl.ToLower()); + var existingElement = items.FirstOrDefault(x => + x.Url.ToLower() == cleanUrl.ToLower() || x.OriginUrl?.ToLower() == cleanUrl.ToLower()); + if (existingElement == null) + { + var addToWallabag = true; + foreach (var p in settings.excludedDomains) + { + r = new Regex(p.pattern, RegexOptions.IgnoreCase); + m = r.Match(cleanUrl); + + if (m.Success) + { + addToWallabag = false; + break; + } + } + + if (addToWallabag && !updatedWallabags.ContainsKey(bookmark.Url)) + { + updatedWallabags.Add(cleanUrl, + bookmark.TagNames.Where(x => !x.Equals(tagName, StringComparison.OrdinalIgnoreCase))); + } + } + } + } + else + { + logger.LogInformation($"no bookmarks found"); + } + + if (updatedWallabags.Count() > 0) + { + logger.LogInformation($"Detected {updatedWallabags.Count()} bookmarks... Start syncing"); + + foreach (var (url, tags) in updatedWallabags) + { + var result = await destinationService.AddEntryByUrl(url, tags); + + if (result.ReadingTime == 0) + { + wallabagToRemove.Add(result.Id); + } + } + + logger.LogInformation($"{updatedWallabags.Count()} bookmarks synced"); + } + } + } +} \ No newline at end of file diff --git a/src/Wallabag/Options/WorkerSettings.cs b/src/Wallabag/Options/WorkerSettings.cs new file mode 100644 index 0000000..d0f868f --- /dev/null +++ b/src/Wallabag/Options/WorkerSettings.cs @@ -0,0 +1,10 @@ +namespace Wallabag.Options; + +public class WorkerSettings +{ + public const string Position = "Worker"; + + public int Intervall { get; set; } = 0; + + public string SyncTag { get; set; } = "readlater"; +} \ No newline at end of file diff --git a/src/Wallabag/Program.cs b/src/Wallabag/Program.cs new file mode 100644 index 0000000..d91725b --- /dev/null +++ b/src/Wallabag/Program.cs @@ -0,0 +1,13 @@ +using Wallabag; + +IHost host = Host.CreateDefaultBuilder(args) + .ConfigureServices((ctx, services) => + { + services.Add_Wallabag_HttpClient(ctx.Configuration); + services.Add_Linkding_HttpClient(ctx.Configuration); + services.Add_Wallabag_Worker(ctx.Configuration); + services.AddHostedService(); + }) + .Build(); + +await host.RunAsync(); \ No newline at end of file diff --git a/src/Wallabag/Properties/launchSettings.json b/src/Wallabag/Properties/launchSettings.json new file mode 100644 index 0000000..d531d73 --- /dev/null +++ b/src/Wallabag/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "WallabagWorker": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/Wallabag/Settings/SettingsService.cs b/src/Wallabag/Settings/SettingsService.cs new file mode 100644 index 0000000..96b6e9b --- /dev/null +++ b/src/Wallabag/Settings/SettingsService.cs @@ -0,0 +1,60 @@ +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Wallabag.Settings; + +public class SettingYaml +{ + public List excludedDomains { get; set; } = new (); +} + +public class ExcludedDomainPattern +{ + public string name { get; set; } + public string pattern { get; set; } +} + +public class SettingsService +{ + private const string fileName = "data/config.yml"; + + private static SettingYaml _settings = null; + + public static SettingYaml Settings + { + get + { + if (_settings == null) + { + Initialize(); + } + + return _settings; + } + private set + { + _settings = value; + } + } + + private static void Initialize() + { + var filePath = Path.Combine(Environment.CurrentDirectory, fileName); + var fileInfo = new FileInfo(filePath); + + if (fileInfo.Exists) + { + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) // see height_in_inches in sample yml + .Build(); + + var yml = File.ReadAllText(fileInfo.FullName); + + Settings = deserializer.Deserialize(yml); + } + else + { + Settings = new SettingYaml(); + } + } +} \ No newline at end of file diff --git a/src/Wallabag/Wallabag.csproj b/src/Wallabag/Wallabag.csproj new file mode 100644 index 0000000..bf5a720 --- /dev/null +++ b/src/Wallabag/Wallabag.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + enable + enable + dotnet-WallabagWorker-D5E52F5F-C642-4BF8-9FD7-8C6C417B0D3A + Linux + + + + + + + + + + + + diff --git a/src/Wallabag/Worker.cs b/src/Wallabag/Worker.cs new file mode 100644 index 0000000..8d14838 --- /dev/null +++ b/src/Wallabag/Worker.cs @@ -0,0 +1,96 @@ +using Core.Abstraction; +using Core.Entities.Linkding; +using Core.Handler; +using Linkding.Client; +using Linkding.Client.Options; +using Microsoft.Extensions.Options; +using Wallabag.Client; +using Wallabag.Client.Options; +using Wallabag.Options; + +namespace Wallabag; + +public class Worker : BackgroundService +{ + private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly ILogger _logger; + private readonly LinkdingService _linkdingService; + private readonly WallabagService _wallabagService; + private readonly LinkdingSettings _linkdingSettings; + private readonly WallabagSettings _wallabagSettings; + private readonly WorkerSettings _settings; + private readonly IConfiguration _configuration; + + + public Worker(ILogger logger, LinkdingService linkdingService, WallabagService wallabagService, + IOptions linkdingSettings, IOptions settings, IOptions wallabagSettings, IConfiguration configuration, IHostApplicationLifetime hostApplicationLifetime) + { + _logger = logger; + _linkdingService = linkdingService; + _wallabagService = wallabagService; + _configuration = configuration; + _hostApplicationLifetime = hostApplicationLifetime; + _wallabagSettings = wallabagSettings.Value; + _settings = settings.Value; + _linkdingSettings = linkdingSettings.Value; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); + + await RunSyncWallabag(); + int delay = _settings.Intervall * 60000; + + if (delay > 0) + { + _logger.LogInformation($"Worker paused for: {_settings.Intervall} minutes"); + + await Task.Delay(delay, stoppingToken); + } + else + { + _logger.LogInformation($"Delay was set to '0' --> stopping worker"); + _hostApplicationLifetime.StopApplication(); + } + } + } + + public async Task RunSyncWallabag() + { + if (!string.IsNullOrEmpty(_wallabagSettings.Url)) + { + + _logger.LogInformation($"Starting updating bookmarks for {_linkdingSettings.Url}"); + _logger.LogInformation("Collectin LinkdingService Handler"); + var wallabagHandlers = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(s => s.GetTypes()) + .Where(p => typeof(ISyncTaskHandler).IsAssignableFrom(p) && p.IsClass); + + if (wallabagHandlers != null && wallabagHandlers.Count() > 0) + { + var wallabags = await _wallabagService.GetEntries(); + + foreach (var handler in wallabagHandlers) + { + ISyncTaskHandler handlerInstance = null; + try + { + handlerInstance = (ISyncTaskHandler) Activator.CreateInstance(handler); + + await handlerInstance.ProcessAsync(wallabags, _wallabagService, _linkdingService, _logger, _configuration); + } + catch (Exception e) + { + Console.WriteLine(e); + throw; + } + } + } + + _logger.LogInformation($"no bookmarks found in {_linkdingSettings.Url}"); + } + } +} \ No newline at end of file diff --git a/src/Wallabag/appsettings.Development.json b/src/Wallabag/appsettings.Development.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/src/Wallabag/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/Wallabag/appsettings.json b/src/Wallabag/appsettings.json new file mode 100644 index 0000000..07c8e4c --- /dev/null +++ b/src/Wallabag/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} \ No newline at end of file diff --git a/src/Wallabag/config.yml b/src/Wallabag/config.yml new file mode 100644 index 0000000..6bce18a --- /dev/null +++ b/src/Wallabag/config.yml @@ -0,0 +1,7 @@ +excludedDomains: + - name: youtube + pattern: https://[[a-zA-Z0-9]+\.]?(youtube)\.com(?:/.*)? + - name: ebay + pattern: https://[[a-zA-Z0-9]+\.]?(ebay)\.(com|de|fr)(?:/.*)? + - name: amazon + pattern: https://[[a-zA-Z0-9]+\.]?(amazon)\.(com|de|fr)(?:/.*)? \ No newline at end of file