tpeetz/kontor
1
0
Fork 0
Code Issues 7 Pull Requests 2 Actions Packages Projects Releases 3 Wiki Activity

add loading links form file
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 4s
Details

Browse Source
This commit is contained in:
Thomas Peetz
2026-10-07 18:50:09 +02:00
parent 980eda29af
commit 094dffb1c7
9 changed files with 384 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-gin/go.mod
+1
View File
@@ -32,6 +32,7 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.3 // indirect
github.com/go-stomp/stomp v2.1.4+incompatible // indirect
github.com/go-stomp/stomp/v3 v3.1.5 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goccy/go-json v0.10.6 // indirect
kontor-gin/go.sum
+2
View File
@@ -36,6 +36,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8=
github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc=
github.com/go-stomp/stomp v2.1.4+incompatible h1:D3SheUVDOz9RsjVWkoh/1iCOwD0qWjyeTZMUZ0EXg2Y=
github.com/go-stomp/stomp v2.1.4+incompatible/go.mod h1:VqCtqNZv1226A1/79yh+rMiFUcfY3R109np+7ke4n0c=
github.com/go-stomp/stomp/v3 v3.1.5 h1:Pikz1OSusmSKUm5mRKYfXQZaDatfZ+EnBBA1JJ2xENQ=
github.com/go-stomp/stomp/v3 v3.1.5/go.mod h1:ztzZej6T2W4Y6FlD+Tb5n7HQP3/O5UNQiuC169pIp10=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
kontor-gin/internal/api/data.go
+244
View File
@@ -0,0 +1,244 @@
package api
import (
"encoding/json"
"fmt"
"kontor-api-gin/internal/models"
"log/slog"
"os"
"resty.dev/v3"
)
type Login struct {
Email string
Password string
}
type LoginResponse struct {
Token string
}
type ExportData struct {
Sport models.SportDTOList `json:"sport"`
Player []models.PlayerDTO `json:"player"`
Team []models.TeamDTO `json:"team"`
Position []models.PositionDTO `json:"field_position"`
Rooster []models.RoosterDTO `json:"rooster"`
Vendor []models.VendorDTO `json:"vendor"`
CardSet []models.CardSetDTO `json:"card_set"`
Card []models.CardDTO `json:"card"`
Artist []models.ArtistDTO `json:"artist"`
Publisher []models.PublisherDTO `json:"publisher"`
Worktype []models.WorkTypeDTO `json:"worktype"`
Comic []models.ComicDTO `json:"comic"`
Volume []models.VolumeDTO `json:"volume"`
StoryArc []models.StoryArcDTO `json:"story_arc"`
Issue []models.IssueDTO `json:"issue"`
TradePaperback []models.TradePaperbackDTO `json:"trade_paperback"`
ComicWork []models.ComicworkDTO `json:"comic_work"`
IssueWork []models.IssueworkDTO `json:"issue_work"`
Article []models.ArticleDTO `json:"article"`
BookshelfPublisher []models.BookshelfPublisherDTO `json:"bookshelf_publisher"`
Book []models.BookDTO `json:"book"`
Author []models.AuthorDTO `json:"author"`
ArticleAuthor []models.ArticleAuthorDTO `json:"article_author"`
BookAuthor []models.BookAuthorDTO `json:"book_author"`
MediaArticle []models.MediaArticleDTO `json:"media_article"`
MediaVideo []models.MediaVideoDTO `json:"media_video"`
MediaLoFi []models.MediaLoFiDTO `json:"media_lofi"`
MediaFile []models.MediaFileDTO `json:"media_file"`
MediaActor []models.MediaActorDTO `json:"media_actor"`
MediaActorFile []models.MediaActorFileDTO `json:"media_actor_file"`
Profile []models.ProfileDTO `json:"profile"`
Permission []models.PermissionDTO `json:"permission"`
Assignment []models.AssignmentDTO `json:"assignment"`
Token []models.TokenDTO `json:"token"`
MailAccount []models.MailAccountDTO `json:"mail_account"`
Mail []models.MailDTO `json:"mail"`
}
func (e *ExportData) GetMap() map[string]any {
dataMap := make(map[string]any)
for path, value := range Registry {
switch path {
case "tysc/sports":
dataMap["sports"] = value
}
}
return dataMap
}
var (
Sports models.SportDTOList
Players []models.PlayerDTO
Teams []models.TeamDTO
Positions []models.PositionDTO
Roosters []models.RoosterDTO
Vendors []models.VendorDTO
Cardsets []models.CardSetDTO
Cards []models.CardDTO
Artists []models.ArtistDTO
Publishers []models.PublisherDTO
Worktypes []models.WorkTypeDTO
Comics []models.ComicDTO
Volumes []models.VolumeDTO
Storyarcs []models.StoryArcDTO
Issues []models.IssueDTO
Tradepaperbacks []models.TradePaperbackDTO
Comicworks []models.ComicworkDTO
Issueworks []models.IssueworkDTO
Articles []models.ArticleDTO
Bookshelf_publishers []models.BookshelfPublisherDTO
Books []models.BookDTO
Authors []models.AuthorDTO
Article_authors []models.ArticleAuthorDTO
Book_authors []models.BookAuthorDTO
Mediaarticles []models.MediaArticleDTO
Mediavideos []models.MediaVideoDTO
Mediaactors []models.MediaActorDTO
Mediaactorfiles []models.MediaActorFileDTO
Mediafiles []models.MediaFileDTO
Medialofis []models.MediaLoFiDTO
Profiles []models.ProfileDTO
Permissions []models.PermissionDTO
Assignments []models.AssignmentDTO
Tokens []models.TokenDTO
Mailaccounts []models.MailAccountDTO
Mail []models.MailDTO
)
var Registry map[string]any
func GetData(server, apiVersion string) (data *ExportData, err error) {
var login LoginResponse
client := resty.New()
PostJSON(client, fmt.Sprintf("%s/%s/%s", server, "api", "login"), &login)
for path, value := range Registry {
GetJSON(client, login.Token, fmt.Sprintf("%s/%s/%s", server, apiVersion, path), value)
}
export := ExportData{
Sport: Sports,
Player: Players,
Team: Teams,
Position: Positions,
Rooster: Roosters,
Vendor: Vendors,
CardSet: Cardsets,
Card: Cards,
Artist: Artists,
Publisher: Publishers,
Worktype: Worktypes,
Comic: Comics,
Volume: Volumes,
StoryArc: Storyarcs,
Issue: Issues,
TradePaperback: Tradepaperbacks,
ComicWork: Comicworks,
IssueWork: Issueworks,
Article: Articles,
BookshelfPublisher: Bookshelf_publishers,
Book: Books,
Author: Authors,
ArticleAuthor: Article_authors,
BookAuthor: Book_authors,
MediaArticle: Mediaarticles,
MediaVideo: Mediavideos,
MediaActor: Mediaactors,
MediaActorFile: Mediaactorfiles,
MediaFile: Mediafiles,
MediaLoFi: Medialofis,
Profile: Profiles,
Permission: Permissions,
Assignment: Assignments,
Token: Tokens,
MailAccount: Mailaccounts,
Mail: Mail,
}
defer client.Close()
return &export, nil
}
func WriteJson(data *ExportData, fileName string) error {
jsonData, err := json.MarshalIndent(data, "", " ")
if err != nil {
slog.Info("JSON MarshalIndent failed", "Error:", err)
return err
}
// Write bytes to file
err = os.WriteFile(fileName, jsonData, 0644)
if err != nil {
slog.Info("Error writing file:", "Error", err)
return err
}
return nil
}
func GetJSON(client *resty.Client, token string, url string, v any) {
_, err := client.R().
SetAuthToken(token).
SetHeader("Accept", "application/json").
SetResult(v).
Get(url)
if err != nil {
slog.Info(url, "error", err.Error())
}
}
func PostJSON(client *resty.Client, url string, v any) {
_, err := client.R().
SetBody(Login{
Email: "thomas.peetz@thpeetz.de",
Password: "admin",
}).
SetHeader("Accept", "application/json").
SetResult(v).
Post(url)
if err != nil {
slog.Info("Health", "error", err.Error())
}
slog.Info(url, "token", v)
}
func init() {
Registry = map[string]any{
"tysc/sports": &Sports,
"tysc/players": &Players,
"tysc/teams": &Teams,
"tysc/positions": &Positions,
"tysc/roosters": &Roosters,
"tysc/vendors": &Vendors,
"tysc/cardsets": &Cardsets,
"tysc/cards": &Cards,
"comics/artists": &Artists,
"comics/publishers": &Publishers,
"comics/worktypes": &Worktypes,
"comics/comics": &Comics,
"comics/volumes": &Volumes,
"comics/storyarcs": &Storyarcs,
"comics/issues": &Issues,
"comics/tradepaperbacks": &Tradepaperbacks,
"comics/comicworks": &Comicworks,
"comics/issueworks": &Issueworks,
"bookshelf/articles": &Articles,
"bookshelf/publishers": &Bookshelf_publishers,
"bookshelf/books": &Books,
"bookshelf/authors": &Authors,
"bookshelf/articleauthors": &Article_authors,
"bookshelf/bookauthors": &Book_authors,
"media/articles": &Mediaarticles,
"media/videos": &Mediavideos,
"media/actors": &Mediaactors,
"media/files": &Mediafiles,
"media/actorfiles": &Mediaactorfiles,
"media/lofis": &Medialofis,
"user/profiles": &Profiles,
"user/permissions": &Permissions,
"user/assignments": &Assignments,
"user/tokens": &Tokens,
"admin/mailaccounts": &Mailaccounts,
"user/mail": &Mail,
}
}
kontor-gin/internal/cmd/add-link.go
+23 -3
View File
@@ -1,12 +1,18 @@
package cmd
import (
"encoding/json"
"log"
"log/slog"
"github.com/JanikL/go-artemis/artemis"
"github.com/go-stomp/stomp/v3"
"github.com/spf13/cobra"
)
type Link struct {
Url string `json:"url"`
}
var addLinkCmd = &cobra.Command{
Use: "add-link",
Short: "Add Link",
@@ -14,8 +20,22 @@ var addLinkCmd = &cobra.Command{
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
slog.Info("add-link kontor started")
sender := artemis.Sender{Addr: "localhost:61616", Dest: "add-link", Enc: artemis.EncodingJson}
err := sender.Send("{ \"url\": \"https://example.com\" }")
conn, err := stomp.Dial("tcp", "localhost:61616",
stomp.ConnOpt.Login("artemis", "artemis"), // Provide username and password
)
if err != nil {
log.Fatalf("failed to connect to Artemis: %s", err)
}
defer conn.Disconnect()
// sender := artemis.Sender{Addr: "localhost:61616", Dest: "add-link", Enc: artemis.EncodingJson}
link := &Link{Url: args[0]}
b, jsonErr := json.Marshal(link)
if jsonErr != nil {
slog.Info("Json encoding failed", "error", jsonErr.Error())
}
err = conn.Send("/queue/add-link", "application/json", b)
if err != nil {
slog.Info("send failed", "error", err.Error())
}
kontor-gin/internal/cmd/add-links.go
+35 -3
View File
@@ -1,18 +1,50 @@
package cmd
import (
"encoding/json"
"fmt"
"kontor-api-gin/internal/helpers"
"log/slog"
"os"
"github.com/go-stomp/stomp/v3"
"github.com/spf13/cobra"
)
var addLinksCmd = &cobra.Command{
Use: "add-links",
Use: "add-links [file]",
Short: "Add Links",
Long: "Add Links",
// Args: cobra.ExactArgs(1),
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
slog.Info("add Links")
conn, err := stomp.Dial("tcp", "localhost:61616",
stomp.ConnOpt.Login("artemis", "artemis"), // Provide username and password
)
if err != nil {
slog.Info("failed to connect to Artemis: %s", err)
os.Exit(1)
}
defer conn.Disconnect()
fileName := args[0]
slog.Info(fmt.Sprintf("add Links from %s", fileName))
lines, err := helpers.LinesInFile(fileName)
if err != nil {
slog.Warn("add links failed", "error", err)
os.Exit(1)
}
for _, line := range lines {
b, jsonErr := json.Marshal(line)
if jsonErr != nil {
slog.Info("Json encoding failed", "error", jsonErr.Error())
}
err = conn.Send("media.link", "application/json", b)
if err != nil {
slog.Info("send failed", "error", err.Error())
}
}
},
}
kontor-gin/internal/cmd/export.go
+5 -166
View File
@@ -1,11 +1,9 @@
package cmd
import (
"encoding/json"
"fmt"
"kontor-api-gin/internal/models"
"kontor-api-gin/internal/api"
"log/slog"
"os"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -32,172 +30,13 @@ var exportCmd = &cobra.Command{
client := resty.New()
PostJSON(client, fmt.Sprintf("%s/%s/%s", server, "api", "login"), &login)
type ExportData struct {
Sport []models.SportDTO `json:"sport"`
Player []models.PlayerDTO `json:"player"`
Team []models.TeamDTO `json:"team"`
Position []models.PositionDTO `json:"field_position"`
Rooster []models.RoosterDTO `json:"rooster"`
Vendor []models.VendorDTO `json:"vendor"`
CardSet []models.CardSetDTO `json:"card_set"`
Card []models.CardDTO `json:"card"`
Artist []models.ArtistDTO `json:"artist"`
Publisher []models.PublisherDTO `json:"publisher"`
Worktype []models.WorkTypeDTO `json:"worktype"`
Comic []models.ComicDTO `json:"comic"`
Volume []models.VolumeDTO `json:"volume"`
StoryArc []models.StoryArcDTO `json:"story_arc"`
Issue []models.IssueDTO `json:"issue"`
TradePaperback []models.TradePaperbackDTO `json:"trade_paperback"`
ComicWork []models.ComicworkDTO `json:"comic_work"`
IssueWork []models.IssueworkDTO `json:"issue_work"`
Article []models.ArticleDTO `json:"article"`
BookshelfPublisher []models.BookshelfPublisherDTO `json:"bookshelf_publisher"`
Book []models.BookDTO `json:"book"`
Author []models.AuthorDTO `json:"author"`
ArticleAuthor []models.ArticleAuthorDTO `json:"article_author"`
BookAuthor []models.BookAuthorDTO `json:"book_author"`
MediaArticle []models.MediaArticleDTO `json:"media_article"`
MediaVideo []models.MediaVideoDTO `json:"media_video"`
MediaLoFi []models.MediaLoFiDTO `json:"media_lofi"`
MediaFile []models.MediaFileDTO `json:"media_file"`
MediaActor []models.MediaActorDTO `json:"media_actor"`
MediaActorFile []models.MediaActorFileDTO `json:"media_actor_file"`
Profile []models.ProfileDTO `json:"profile"`
Permission []models.PermissionDTO `json:"permission"`
Assignment []models.AssignmentDTO `json:"assignment"`
Token []models.TokenDTO `json:"token"`
MailAccount []models.MailAccountDTO `json:"mail_account"`
Mail []models.MailDTO `json:"mail"`
}
var (
sports []models.SportDTO
players []models.PlayerDTO
teams []models.TeamDTO
positions []models.PositionDTO
roosters []models.RoosterDTO
vendors []models.VendorDTO
cardsets []models.CardSetDTO
cards []models.CardDTO
artists []models.ArtistDTO
publishers []models.PublisherDTO
worktypes []models.WorkTypeDTO
comics []models.ComicDTO
volumes []models.VolumeDTO
storyarcs []models.StoryArcDTO
issues []models.IssueDTO
tradepaperbacks []models.TradePaperbackDTO
comicworks []models.ComicworkDTO
issueworks []models.IssueworkDTO
articles []models.ArticleDTO
bookshelf_publishers []models.BookshelfPublisherDTO
books []models.BookDTO
autors []models.AuthorDTO
article_authors []models.ArticleAuthorDTO
book_authors []models.BookAuthorDTO
mediaarticles []models.MediaArticleDTO
mediavideos []models.MediaVideoDTO
mediaactors []models.MediaActorDTO
mediaactorfiles []models.MediaActorFileDTO
mediafiles []models.MediaFileDTO
medialofis []models.MediaLoFiDTO
profiles []models.ProfileDTO
permissions []models.PermissionDTO
assignments []models.AssignmentDTO
tokens []models.TokenDTO
mailaccounts []models.MailAccountDTO
mail []models.MailDTO
)
registry := map[string]any{
"tysc/sports": &sports,
"tysc/players": &players,
"tysc/teams": &teams,
"tysc/positions": &positions,
"tysc/roosters": &roosters,
"tysc/vendors": &vendors,
"tysc/cardsets": &cardsets,
"tysc/cards": &cards,
"comics/artists": &artists,
"comics/publishers": &publishers,
"comics/worktypes": &worktypes,
"comics/comics": &comics,
"comics/volumes": &volumes,
"comics/storyarcs": &storyarcs,
"comics/issues": &issues,
"comics/tradepaperbacks": &tradepaperbacks,
"comics/comicworks": &comicworks,
"comics/issueworks": &issueworks,
"bookshelf/articles": &articles,
"bookshelf/publishers": &bookshelf_publishers,
"bookshelf/books": &books,
"bookshelf/authors": &autors,
"bookshelf/articleauthors": &article_authors,
"bookshelf/bookauthors": &book_authors,
"media/articles": &mediaarticles,
"media/videos": &mediavideos,
"media/actors": &mediaactors,
"media/files": &mediafiles,
"media/actorfiles": &mediaactorfiles,
"media/lofis": &medialofis,
"user/profiles": &profiles,
"user/permissions": &permissions,
"user/assignments": &assignments,
"user/tokens": &tokens,
"admin/mailaccounts": &mailaccounts,
"user/mail": &mail,
}
for path, value := range registry {
for path, value := range api.Registry {
GetJSON(client, login.Token, fmt.Sprintf("%s/%s/%s", server, apiVersion, path), value)
}
export := ExportData{
Sport: sports,
Player: players,
Team: teams,
Position: positions,
Rooster: roosters,
Vendor: vendors,
CardSet: cardsets,
Card: cards,
Artist: artists,
Publisher: publishers,
Worktype: worktypes,
Comic: comics,
Volume: volumes,
StoryArc: storyarcs,
Issue: issues,
TradePaperback: tradepaperbacks,
ComicWork: comicworks,
IssueWork: issueworks,
Article: articles,
BookshelfPublisher: bookshelf_publishers,
Book: books,
Author: autors,
ArticleAuthor: article_authors,
BookAuthor: book_authors,
MediaArticle: mediaarticles,
MediaVideo: mediavideos,
MediaActor: mediaactors,
MediaActorFile: mediaactorfiles,
MediaFile: mediafiles,
MediaLoFi: medialofis,
Profile: profiles,
Permission: permissions,
Assignment: assignments,
Token: tokens,
MailAccount: mailaccounts,
Mail: mail,
}
defer client.Close()
jsonData, err := json.MarshalIndent(export, "", " ")
export, _ := api.GetData(server, apiVersion)
err := api.WriteJson(export, filename)
if err != nil {
slog.Info("JSON MarshalIndent failed", "Error:", err)
}
// Write bytes to file
err = os.WriteFile(filename, jsonData, 0644)
if err != nil {
slog.Info("Error writing file:", "Error", err)
slog.Info("WriteJon failed", "Error:", err)
}
slog.Info("export kontor finished")
},
kontor-gin/internal/cmd/sync.go
+15
View File
@@ -1,7 +1,9 @@
package cmd
import (
"kontor-api-gin/internal/api"
"log/slog"
"reflect"
"github.com/spf13/cobra"
)
@@ -13,6 +15,19 @@ var syncCmd = &cobra.Command{
// Args: cobra.ExactArgs(2),
Run: func(cmd *cobra.Command, args []string) {
slog.Info("sync kontor")
sourceData, _ := api.GetData(server, apiVersion)
targetData, _ := api.GetData("https://kontor-gin.thpeetz.de", apiVersion)
slog.Info("Size of targetData", "size", reflect.TypeOf(targetData).Size())
// find missing items in target and collect items not available in source
for table, item_list := range sourceData.GetMap() {
slog.Info("iterate sourceData", "table", table, "count", reflect.TypeOf(item_list).Size())
switch table {
case "sports":
sourceData.Sport.CompareIds(&targetData.Sport)
}
}
},
}
kontor-gin/internal/helpers/file.go
+27
View File
@@ -0,0 +1,27 @@
package helpers
import (
"bufio"
"os"
)
func LinesInFile(fileName string) ([]string, error) {
f, err := os.Open(fileName)
if err != nil {
return nil, err
}
defer f.Close()
// Create new Scanner.
scanner := bufio.NewScanner(f)
result := []string{}
// Use Scan.
for scanner.Scan() {
line := scanner.Text()
// Append line to result.
result = append(result, line)
}
if err := scanner.Err(); err != nil {
return nil, err
}
return result, nil
}
kontor-gin/internal/models/sport.go
+32
View File
@@ -1,7 +1,9 @@
package models
import (
"fmt"
"kontor-api-gin/internal/helpers"
"log/slog"
"time"
)
@@ -27,6 +29,36 @@ type SportDTO struct {
Name string `json:"name" binding:"required"`
}
type SportDTOList []SportDTO
func (sportList *SportDTOList) CompareIds(target *SportDTOList) (missing, obsolete []string) {
sources := make(map[string]SportDTO, len(*sportList))
targets := make(map[string]SportDTO, len(*target))
for _, source := range *sportList {
sources[source.ID] = source
}
for _, target := range *target {
targets[target.ID] = target
}
// iterate source IDs
for id := range sources {
if _, ok := targets[id]; ok {
slog.Info(fmt.Sprintf("ID %s found in both lists", id))
} else {
missing = append(missing, id)
}
}
for id := range targets {
if _, ok := sources[id]; ok {
slog.Info(fmt.Sprintf("ID %s found in both lists", id))
} else {
obsolete = append(obsolete, id)
}
}
slog.Info("CompareIds", "missing", missing, "obsolete", obsolete)
return missing, obsolete
}
func ConvertSportsToDtos(sports []Sport) []SportDTO {
dtos := make([]SportDTO, len(sports))
for i, sport := range sports {