From 76d91dd506348eed286d54a23a0a9fd3353eadd7 Mon Sep 17 00:00:00 2001 From: Thomas Peetz Date: Mon, 7 Apr 2025 19:34:08 +0200 Subject: [PATCH] add import view --- springboot/.gitignore | 1 + springboot/build.gradle | 5 +- springboot/gradle/libs.versions.toml | 8 ++ .../kontor/admin/SetupModuleAdmin.java | 4 +- .../kontor/admin/data/MetaDataTable.java | 12 ++ .../AuthorizationMatrixRepository.java | 6 +- .../MailAccountRepository.java | 2 +- .../MetaDataColumnRepository.java | 4 +- .../MetaDataTableRepository.java | 3 +- .../ModuleDataRepository.java | 3 +- .../{data => repository}/RoleRepository.java | 3 +- .../{data => repository}/UserRepository.java | 3 +- .../kontor/admin/services/AdminService.java | 6 +- .../services/KontorUserDetailsService.java | 3 + .../kontor/admin/services/MailService.java | 2 +- .../admin/services/MetaDataService.java | 63 +++++++++- .../kontor/admin/services/ModuleService.java | 2 +- .../kontor/common/data/AbstractEntity.java | 11 ++ .../kontor/common/views/MainLayout.java | 5 +- .../data/services/DataManagementService.java | 40 +++++++ .../kontor/data/views/DataManagementView.java | 49 ++++++++ .../thpeetz/kontor/data/views/ImportArea.java | 111 ++++++++++++++++++ .../thpeetz/kontor/data/views/UploadArea.java | 58 +++++++++ springboot/src/main/resources/application.yml | 4 + 24 files changed, 387 insertions(+), 21 deletions(-) rename springboot/src/main/java/de/thpeetz/kontor/admin/{data => repository}/AuthorizationMatrixRepository.java (61%) rename springboot/src/main/java/de/thpeetz/kontor/admin/{data => repository}/MailAccountRepository.java (82%) rename springboot/src/main/java/de/thpeetz/kontor/admin/{data => repository}/MetaDataColumnRepository.java (80%) rename springboot/src/main/java/de/thpeetz/kontor/admin/{data => repository}/MetaDataTableRepository.java (68%) rename springboot/src/main/java/de/thpeetz/kontor/admin/{data => repository}/ModuleDataRepository.java (84%) rename springboot/src/main/java/de/thpeetz/kontor/admin/{data => repository}/RoleRepository.java (87%) rename springboot/src/main/java/de/thpeetz/kontor/admin/{data => repository}/UserRepository.java (85%) create mode 100644 springboot/src/main/java/de/thpeetz/kontor/data/services/DataManagementService.java create mode 100644 springboot/src/main/java/de/thpeetz/kontor/data/views/DataManagementView.java create mode 100644 springboot/src/main/java/de/thpeetz/kontor/data/views/ImportArea.java create mode 100644 springboot/src/main/java/de/thpeetz/kontor/data/views/UploadArea.java diff --git a/springboot/.gitignore b/springboot/.gitignore index 0294d4c..9896012 100644 --- a/springboot/.gitignore +++ b/springboot/.gitignore @@ -30,3 +30,4 @@ kontorHSQLDB* src/main/resources/application-local.properties src/main/resources/application-prod.properties src/main/resources/application-*.yml +/uploaded-files/ diff --git a/springboot/build.gradle b/springboot/build.gradle index 1d7ad02..01b41ea 100644 --- a/springboot/build.gradle +++ b/springboot/build.gradle @@ -62,7 +62,10 @@ dependencies { implementation 'com.h2database:h2' implementation libs.hsqldb runtimeOnly 'org.mariadb.jdbc:mariadb-java-client' - implementation 'com.sun.mail:javax.mail:1.6.2' + implementation libs.mail + implementation libs.jackson + implementation libs.gson + implementation libs.json implementation 'org.hibernate.orm:hibernate-community-dialects' testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' diff --git a/springboot/gradle/libs.versions.toml b/springboot/gradle/libs.versions.toml index 6ec0bbe..8f74e77 100644 --- a/springboot/gradle/libs.versions.toml +++ b/springboot/gradle/libs.versions.toml @@ -20,6 +20,10 @@ springboot = "3.2.5" springdependencies = "1.1.4" vaadin = "24.3.8" lombok = "8.6" +gson = "2.9.0" +jackson = "2.16.1" +json_simple = "1.1.1" +mail = "1.6.2" [libraries] args4j = { module = "args4j:args4j", version.ref = "args4j" } @@ -31,6 +35,10 @@ mockito = { module = "org.mockito:mockito-all", version.ref = "mockito" } picocli = { module = "info.picocli:picocli", version.ref = "picoli" } slf4j = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } hsqldb = { module = "org.hsqldb:hsqldb", version.ref = "hsqldb" } +gson = { module = "com.google.code.gson:gson", version.ref = "gson" } +jackson = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } +json = { module = "com.googlecode.json-simple:json-simple", version.ref ="json_simple" } +mail = { module = "com.sun.mail:javax.mail", version.ref ="mail" } sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite" } vaadin-bom = { module = "com.vaadin:vaadin-bom", version.ref = "vaadin" } asciidoctorGradleJvmGems = { module = "org.asciidoctor:asciidoctor-gradle-jvm-gems", version.ref= "asciidoctor" } diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java b/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java index a819bb2..1577e7a 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java @@ -1,7 +1,9 @@ package de.thpeetz.kontor.admin; -import com.vaadin.flow.component.page.Meta; import de.thpeetz.kontor.admin.data.*; +import de.thpeetz.kontor.admin.repository.AuthorizationMatrixRepository; +import de.thpeetz.kontor.admin.repository.MailAccountRepository; +import de.thpeetz.kontor.admin.repository.UserRepository; import de.thpeetz.kontor.admin.services.AdminService; import de.thpeetz.kontor.admin.services.MetaDataService; import de.thpeetz.kontor.mailclient.data.MailAccount; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataTable.java b/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataTable.java index 639666d..0c8f267 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataTable.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataTable.java @@ -5,10 +5,15 @@ import jakarta.persistence.*; import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.LinkedList; import java.util.List; +@Slf4j @Entity @Getter @Setter @@ -23,4 +28,11 @@ public class MetaDataTable extends AbstractEntity { @OneToMany(fetch = FetchType.EAGER, mappedBy = "table") private List tableColumns = new LinkedList<>(); + + public void updateTableName(String value) { + if (!this.getTableName().equals(value)) { + this.setTableName(value); + log.info("update tableName"); + } + } } diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/AuthorizationMatrixRepository.java b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/AuthorizationMatrixRepository.java similarity index 61% rename from springboot/src/main/java/de/thpeetz/kontor/admin/data/AuthorizationMatrixRepository.java rename to springboot/src/main/java/de/thpeetz/kontor/admin/repository/AuthorizationMatrixRepository.java index 25d2fb9..3e5c30c 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/AuthorizationMatrixRepository.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/AuthorizationMatrixRepository.java @@ -1,8 +1,10 @@ -package de.thpeetz.kontor.admin.data; +package de.thpeetz.kontor.admin.repository; import java.util.List; -import java.util.UUID; +import de.thpeetz.kontor.admin.data.AuthorizationMatrix; +import de.thpeetz.kontor.admin.data.Role; +import de.thpeetz.kontor.admin.data.User; import org.springframework.data.jpa.repository.JpaRepository; public interface AuthorizationMatrixRepository extends JpaRepository { diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MailAccountRepository.java b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/MailAccountRepository.java similarity index 82% rename from springboot/src/main/java/de/thpeetz/kontor/admin/data/MailAccountRepository.java rename to springboot/src/main/java/de/thpeetz/kontor/admin/repository/MailAccountRepository.java index ffe66ae..1cd3885 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MailAccountRepository.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/MailAccountRepository.java @@ -1,4 +1,4 @@ -package de.thpeetz.kontor.admin.data; +package de.thpeetz.kontor.admin.repository; import de.thpeetz.kontor.mailclient.data.MailAccount; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataColumnRepository.java b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/MetaDataColumnRepository.java similarity index 80% rename from springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataColumnRepository.java rename to springboot/src/main/java/de/thpeetz/kontor/admin/repository/MetaDataColumnRepository.java index f822a14..c0259bf 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataColumnRepository.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/MetaDataColumnRepository.java @@ -1,5 +1,7 @@ -package de.thpeetz.kontor.admin.data; +package de.thpeetz.kontor.admin.repository; +import de.thpeetz.kontor.admin.data.MetaDataColumn; +import de.thpeetz.kontor.admin.data.MetaDataTable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataTableRepository.java b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/MetaDataTableRepository.java similarity index 68% rename from springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataTableRepository.java rename to springboot/src/main/java/de/thpeetz/kontor/admin/repository/MetaDataTableRepository.java index 694ee52..4e31cb4 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/MetaDataTableRepository.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/MetaDataTableRepository.java @@ -1,5 +1,6 @@ -package de.thpeetz.kontor.admin.data; +package de.thpeetz.kontor.admin.repository; +import de.thpeetz.kontor.admin.data.MetaDataTable; import org.springframework.data.jpa.repository.JpaRepository; public interface MetaDataTableRepository extends JpaRepository { diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/ModuleDataRepository.java b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/ModuleDataRepository.java similarity index 84% rename from springboot/src/main/java/de/thpeetz/kontor/admin/data/ModuleDataRepository.java rename to springboot/src/main/java/de/thpeetz/kontor/admin/repository/ModuleDataRepository.java index 484dde3..ace45ef 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/ModuleDataRepository.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/ModuleDataRepository.java @@ -1,5 +1,6 @@ -package de.thpeetz.kontor.admin.data; +package de.thpeetz.kontor.admin.repository; +import de.thpeetz.kontor.admin.data.ModuleData; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/RoleRepository.java b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/RoleRepository.java similarity index 87% rename from springboot/src/main/java/de/thpeetz/kontor/admin/data/RoleRepository.java rename to springboot/src/main/java/de/thpeetz/kontor/admin/repository/RoleRepository.java index 932ae29..0d1b466 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/RoleRepository.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/RoleRepository.java @@ -1,7 +1,8 @@ -package de.thpeetz.kontor.admin.data; +package de.thpeetz.kontor.admin.repository; import java.util.List; +import de.thpeetz.kontor.admin.data.Role; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/data/UserRepository.java b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/UserRepository.java similarity index 85% rename from springboot/src/main/java/de/thpeetz/kontor/admin/data/UserRepository.java rename to springboot/src/main/java/de/thpeetz/kontor/admin/repository/UserRepository.java index 21d93e0..c8c20f3 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/data/UserRepository.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/repository/UserRepository.java @@ -1,7 +1,8 @@ -package de.thpeetz.kontor.admin.data; +package de.thpeetz.kontor.admin.repository; import java.util.List; +import de.thpeetz.kontor.admin.data.User; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/services/AdminService.java b/springboot/src/main/java/de/thpeetz/kontor/admin/services/AdminService.java index 77a6448..c7c854c 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/services/AdminService.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/services/AdminService.java @@ -6,11 +6,11 @@ import java.util.List; import org.springframework.stereotype.Service; import de.thpeetz.kontor.admin.data.AuthorizationMatrix; -import de.thpeetz.kontor.admin.data.AuthorizationMatrixRepository; +import de.thpeetz.kontor.admin.repository.AuthorizationMatrixRepository; import de.thpeetz.kontor.admin.data.Role; -import de.thpeetz.kontor.admin.data.RoleRepository; +import de.thpeetz.kontor.admin.repository.RoleRepository; import de.thpeetz.kontor.admin.data.User; -import de.thpeetz.kontor.admin.data.UserRepository; +import de.thpeetz.kontor.admin.repository.UserRepository; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/services/KontorUserDetailsService.java b/springboot/src/main/java/de/thpeetz/kontor/admin/services/KontorUserDetailsService.java index f76d6df..6a1eead 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/services/KontorUserDetailsService.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/services/KontorUserDetailsService.java @@ -1,6 +1,9 @@ package de.thpeetz.kontor.admin.services; import de.thpeetz.kontor.admin.data.*; +import de.thpeetz.kontor.admin.repository.AuthorizationMatrixRepository; +import de.thpeetz.kontor.admin.repository.RoleRepository; +import de.thpeetz.kontor.admin.repository.UserRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.GrantedAuthority; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/services/MailService.java b/springboot/src/main/java/de/thpeetz/kontor/admin/services/MailService.java index dbe9270..d817197 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/services/MailService.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/services/MailService.java @@ -1,7 +1,7 @@ package de.thpeetz.kontor.admin.services; import de.thpeetz.kontor.mailclient.data.MailAccount; -import de.thpeetz.kontor.admin.data.MailAccountRepository; +import de.thpeetz.kontor.admin.repository.MailAccountRepository; import lombok.extern.slf4j.Slf4j; import java.util.List; diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/services/MetaDataService.java b/springboot/src/main/java/de/thpeetz/kontor/admin/services/MetaDataService.java index d68f7d6..dcc97d6 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/services/MetaDataService.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/services/MetaDataService.java @@ -3,12 +3,16 @@ package de.thpeetz.kontor.admin.services; import org.springframework.stereotype.Service; import de.thpeetz.kontor.admin.data.MetaDataColumn; -import de.thpeetz.kontor.admin.data.MetaDataColumnRepository; +import de.thpeetz.kontor.admin.repository.MetaDataColumnRepository; import de.thpeetz.kontor.admin.data.MetaDataTable; -import de.thpeetz.kontor.admin.data.MetaDataTableRepository; +import de.thpeetz.kontor.admin.repository.MetaDataTableRepository; import lombok.extern.slf4j.Slf4j; +import java.sql.Date; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; @Slf4j @Service @@ -23,6 +27,10 @@ public class MetaDataService { this.metaDataColumnRepository = metaDataColumnRepository; } + public List findAllTables() { + return metaDataTableRepository.findAll(); + } + public MetaDataTable getTable(String tableName) { MetaDataTable table = metaDataTableRepository.findByTableName(tableName); if (table == null) { @@ -34,6 +42,14 @@ public class MetaDataService { return table; } + private void deleteTable(MetaDataTable metaDataTable) { + List columns = metaDataTable.getTableColumns(); + for (MetaDataColumn column: columns) { + metaDataColumnRepository.delete(column); + } + metaDataTableRepository.delete(metaDataTable); + } + public void getColumn(MetaDataTable table, String columnName, String columnSyncName, String columnType, String columnModifier, Integer columnOrder, Boolean isShown, String columnLabel, Boolean showFilter, String filterLabel) { this.getColumn(table, columnName, columnSyncName, columnType, columnModifier, columnOrder, isShown, columnLabel, showFilter, filterLabel, null); } @@ -123,7 +139,46 @@ public class MetaDataService { metaDataColumnRepository.save(metaDataColumn); } - public List findAllTables() { - return metaDataTableRepository.findAll(); + public String importData(Map fields) { + AtomicReference status = new AtomicReference<>("unknown"); + String id = fields.get("id"); + Optional optional = metaDataTableRepository.findById(id); + if (optional.isEmpty()) { + log.info(" not found: {} with {}", id, fields); + status.set(id + "not found"); + MetaDataTable checkExisting = metaDataTableRepository.findByTableName(fields.get("table_name")); + if (checkExisting != null) { + log.info("entry already there with different id ({}), will be deleted", checkExisting.getId()); + deleteTable(checkExisting); + } + MetaDataTable metaDataTable = new MetaDataTable(); + metaDataTable.setId(id); + metaDataTable.setTableName(fields.get("table_name")); + metaDataTableRepository.save(metaDataTable); + } else { + optional.ifPresent( entry -> { + log.info(" found: {}", entry.getTableName()); + updateFields(entry, fields); + metaDataTableRepository.save(entry); + status.set("found"); + }); + } + return status.get(); + } + + private void updateFields(MetaDataTable metaDataTable, Map fields) { + for (Map.Entry entry : fields.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + switch (key) { + case "id", "created_date", "last_modified_date", "version": + break; + case "table_name": + metaDataTable.updateTableName(value); + break; + default: + log.info("field {} is unknown for table {}", key, MetaDataTable.class.getName()); + } + } } } diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/services/ModuleService.java b/springboot/src/main/java/de/thpeetz/kontor/admin/services/ModuleService.java index 7596701..8c5f929 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/services/ModuleService.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/services/ModuleService.java @@ -1,7 +1,7 @@ package de.thpeetz.kontor.admin.services; import de.thpeetz.kontor.admin.data.ModuleData; -import de.thpeetz.kontor.admin.data.ModuleDataRepository; +import de.thpeetz.kontor.admin.repository.ModuleDataRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/springboot/src/main/java/de/thpeetz/kontor/common/data/AbstractEntity.java b/springboot/src/main/java/de/thpeetz/kontor/common/data/AbstractEntity.java index f9cfdaf..182dd45 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/common/data/AbstractEntity.java +++ b/springboot/src/main/java/de/thpeetz/kontor/common/data/AbstractEntity.java @@ -31,4 +31,15 @@ public abstract class AbstractEntity { @LastModifiedDate private Date lastModifiedDate; + + public void updateVersion(String value) { + if (value == null) { + log.info("value not given"); + return; + } + if (version != Integer.valueOf(value).intValue()) { + log.info("update version"); + version = Integer.valueOf(value).intValue(); + } + } } diff --git a/springboot/src/main/java/de/thpeetz/kontor/common/views/MainLayout.java b/springboot/src/main/java/de/thpeetz/kontor/common/views/MainLayout.java index 96f720f..95d60cc 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/common/views/MainLayout.java +++ b/springboot/src/main/java/de/thpeetz/kontor/common/views/MainLayout.java @@ -4,6 +4,7 @@ import com.vaadin.flow.component.applayout.AppLayout; import com.vaadin.flow.component.applayout.DrawerToggle; import com.vaadin.flow.component.html.H1; import com.vaadin.flow.component.html.H2; +import com.vaadin.flow.component.icon.VaadinIcon; import com.vaadin.flow.component.orderedlayout.FlexComponent; import com.vaadin.flow.component.orderedlayout.HorizontalLayout; import com.vaadin.flow.component.orderedlayout.Scroller; @@ -17,15 +18,14 @@ import de.thpeetz.kontor.admin.AdminConstants; import de.thpeetz.kontor.admin.services.AdminService; import de.thpeetz.kontor.bookshelf.BookshelfConstants; import de.thpeetz.kontor.comics.ComicConstants; +import de.thpeetz.kontor.data.views.DataManagementView; import de.thpeetz.kontor.mailclient.views.EmailView; import de.thpeetz.kontor.media.MediaConstants; import de.thpeetz.kontor.security.SecurityService; import de.thpeetz.kontor.tysc.TyscConstants; import lombok.extern.slf4j.Slf4j; -import org.springframework.security.core.GrantedAuthority; import java.util.ArrayList; -import java.util.Collection; @Slf4j public class MainLayout extends AppLayout { @@ -87,6 +87,7 @@ public class MainLayout extends AppLayout { sideNav.addItem(BookshelfConstants.getBookshelfNavigation()); sideNav.addItem(MediaConstants.getMediaNavigation(roles)); sideNav.addItem(new SideNavItem("Emails", EmailView.class)); + sideNav.addItem(new SideNavItem("Data Management", DataManagementView.class, VaadinIcon.DATABASE.create())); securityService.getAuthenticatedUser().ifPresent(user -> { log.info("User {} found", user.getUsername()); boolean isAdmin = user.getAuthorities().stream() diff --git a/springboot/src/main/java/de/thpeetz/kontor/data/services/DataManagementService.java b/springboot/src/main/java/de/thpeetz/kontor/data/services/DataManagementService.java new file mode 100644 index 0000000..27389d3 --- /dev/null +++ b/springboot/src/main/java/de/thpeetz/kontor/data/services/DataManagementService.java @@ -0,0 +1,40 @@ +package de.thpeetz.kontor.data.services; + +import de.thpeetz.kontor.admin.data.MetaDataTable; +import de.thpeetz.kontor.admin.repository.MetaDataTableRepository; +import de.thpeetz.kontor.admin.services.MetaDataService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +@Slf4j +@Service +public class DataManagementService { + + @Autowired + MetaDataTableRepository metaDataTableRepository; + + @Autowired + MetaDataService metaDataService; + + public DataManagementService() { + + } + + public String getEntry(String nodeName, Map fields) { + AtomicReference status = new AtomicReference<>("unknown"); + switch (nodeName) { + case "meta_data_table": + //status.set(metaDataService.importData(fields)); + break; + default: + log.debug("import for {} not implemented", nodeName); + break; + } + return status.get(); + } +} diff --git a/springboot/src/main/java/de/thpeetz/kontor/data/views/DataManagementView.java b/springboot/src/main/java/de/thpeetz/kontor/data/views/DataManagementView.java new file mode 100644 index 0000000..43e0675 --- /dev/null +++ b/springboot/src/main/java/de/thpeetz/kontor/data/views/DataManagementView.java @@ -0,0 +1,49 @@ +package de.thpeetz.kontor.data.views; + +import com.vaadin.flow.component.checkbox.Checkbox; +import com.vaadin.flow.component.orderedlayout.VerticalLayout; +import com.vaadin.flow.router.PageTitle; +import com.vaadin.flow.router.Route; +import com.vaadin.flow.spring.annotation.SpringComponent; +import de.thpeetz.kontor.common.views.MainLayout; +import de.thpeetz.kontor.data.services.DataManagementService; +import jakarta.annotation.security.PermitAll; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Scope; + +import java.io.File; + +@Slf4j +@SpringComponent +@Scope("prototype") +@PermitAll +@Route(value = "common/import", layout = MainLayout.class) +@PageTitle("Import Data | Kontor") +public class DataManagementView extends VerticalLayout { + + DataManagementService service; + + public DataManagementView(DataManagementService service) { + this.service = service; + File uploadFolder = getUploadFolder(); + UploadArea uploadArea = new UploadArea(uploadFolder); + Checkbox pruneTables = new Checkbox("Prune Table"); + ImportArea importArea = new ImportArea(uploadFolder, pruneTables, service); + + uploadArea.getUploadField().addSucceededListener(e -> { + uploadArea.hideErrorField(); + importArea.processFiles(); + uploadArea.getUploadField().clearFileList(); + }); + + add(uploadArea, pruneTables, importArea); + } + + private File getUploadFolder() { + File folder = new File("uploaded-files"); + if (!folder.exists()) { + folder.mkdirs(); + } + return folder; + } +} diff --git a/springboot/src/main/java/de/thpeetz/kontor/data/views/ImportArea.java b/springboot/src/main/java/de/thpeetz/kontor/data/views/ImportArea.java new file mode 100644 index 0000000..831f6a3 --- /dev/null +++ b/springboot/src/main/java/de/thpeetz/kontor/data/views/ImportArea.java @@ -0,0 +1,111 @@ +package de.thpeetz.kontor.data.views; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import com.vaadin.flow.component.checkbox.Checkbox; +import com.vaadin.flow.component.html.H4; +import com.vaadin.flow.component.html.Paragraph; +import com.vaadin.flow.component.orderedlayout.VerticalLayout; +import de.thpeetz.kontor.data.services.DataManagementService; +import lombok.extern.slf4j.Slf4j; +import org.json.simple.parser.ParseException; + +import java.io.*; +import java.util.*; + +@Slf4j +public class ImportArea extends VerticalLayout { + + protected File uploadFolder; + DataManagementService service; + boolean pruneTables = false; + + public ImportArea(File uploadFolder, Checkbox pruneTablesCheckbox, DataManagementService service) { + this.uploadFolder = uploadFolder; + this.service = service; + this.pruneTables = pruneTablesCheckbox.isEnabled(); + setMargin(true); + } + + public void processFiles() { + removeAll(); + if (pruneTables) { + add(new Paragraph("prune all tables")); + } + add(new H4("Process file")); + + for (File file: uploadFolder.listFiles()) { + add(new Paragraph("reading " + file.getName())); + try { + processFile(file); + } catch (ParseException e) { + throw new RuntimeException(e); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private void processFile(File file) throws FileNotFoundException, IOException, ParseException { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode node = objectMapper.readTree(file); + log.info("parsing: {}, {}", node.getNodeType(), node.size()); + Iterator fieldNamesIterator = node.fieldNames(); + while (fieldNamesIterator.hasNext()) { + String fieldName = fieldNamesIterator.next(); + parseRoot(node, fieldName); + } + } + + private void parseRoot(JsonNode parentNode, String field) { + JsonNode node = parentNode.get(field); + String nodeType = String.valueOf(node.getNodeType()); + if (nodeType.equals("ARRAY")) { + log.info(" {}: {}", field, node.size()); + Iterator elements = node.elements(); + while (elements.hasNext()) { + JsonNode subNode = elements.next(); + String status = parseNode(subNode, field); + add(new Paragraph(status)); + } + } else { + log.info("unknown type: {}", nodeType); + } + } + + private String parseNode(JsonNode node, String nodeName) { + Map fields = parseFields(node); + return service.getEntry(nodeName, fields); + } + + private Map parseFields(JsonNode node) { + Map fields = new HashMap<>(); + node.fieldNames().forEachRemaining(field -> { + + JsonNode value = node.get(field); + log.debug("type: {}", value.getNodeType()); + JsonNodeType nodeType = value.getNodeType(); + switch (nodeType) { + case NUMBER: + fields.put(field, String.valueOf(value.intValue())); + break; + case STRING: + String stringValue = value.textValue(); + if (stringValue.contains("\"")) { + log.debug("remove double quotes"); + String stripped = stringValue.substring(1, stringValue.length()-1); + fields.put(field, stripped); + } + fields.put(field, value.textValue()); + break; + default: + log.debug("no implementation for type {}", nodeType.name()); + } + }); + log.debug("fields: {}", fields); + return fields; + } +} diff --git a/springboot/src/main/java/de/thpeetz/kontor/data/views/UploadArea.java b/springboot/src/main/java/de/thpeetz/kontor/data/views/UploadArea.java new file mode 100644 index 0000000..f82efda --- /dev/null +++ b/springboot/src/main/java/de/thpeetz/kontor/data/views/UploadArea.java @@ -0,0 +1,58 @@ +package de.thpeetz.kontor.data.views; + +import com.vaadin.flow.component.html.Span; +import com.vaadin.flow.component.orderedlayout.VerticalLayout; +import com.vaadin.flow.component.upload.MultiFileReceiver; +import com.vaadin.flow.component.upload.Receiver; +import com.vaadin.flow.component.upload.Upload; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; + +@Slf4j +public class UploadArea extends VerticalLayout { + + @Getter + private final Upload uploadField; + private final Span errorField; + + public UploadArea(File uploadFolder) { + uploadField = new Upload(createFileReceiver(uploadFolder)); + uploadField.setMaxFiles(1); + uploadField.setMaxFileSize(2*1024*1024); + + errorField = new Span(); + errorField.setVisible(false); + errorField.getStyle().set("color", "red"); + + uploadField.addFailedListener(e -> showErrorMessage(e.getReason().getMessage())); + uploadField.addFileRejectedListener(e -> showErrorMessage(e.getErrorMessage())); + + add(uploadField, errorField); + } + + public void hideErrorField() { + errorField.setVisible(false); + } + + private Receiver createFileReceiver(File uploadFolder) { + return (MultiFileReceiver) (filename, mimetype) -> { + File file = new File(uploadFolder, filename); + try { + return new FileOutputStream(file); + } + catch (FileNotFoundException fnfe) { + log.info(fnfe.getStackTrace().toString()); + return null; + } + }; + } + + private void showErrorMessage(String message) { + errorField.setVisible(true); + errorField.setText(message); + } +} diff --git a/springboot/src/main/resources/application.yml b/springboot/src/main/resources/application.yml index 696dd47..c884eb5 100644 --- a/springboot/src/main/resources/application.yml +++ b/springboot/src/main/resources/application.yml @@ -17,6 +17,10 @@ spring: mode: never mustache: check-template-location: false + servlet: + multipart: + max-file-size: 10MB + max-request-size: 10MB management: endpoints: web: