add import view

This commit is contained in:
Thomas Peetz
2025-04-07 19:34:08 +02:00
parent 84aff2b7d7
commit 76d91dd506
24 changed files with 387 additions and 21 deletions
+1
View File
@@ -30,3 +30,4 @@ kontorHSQLDB*
src/main/resources/application-local.properties
src/main/resources/application-prod.properties
src/main/resources/application-*.yml
/uploaded-files/
+4 -1
View File
@@ -62,7 +62,10 @@ dependencies {
implementation 'com.h2database:h2'
implementation libs.hsqldb
runtimeOnly 'org.mariadb.jdbc:mariadb-java-client'
implementation 'com.sun.mail:javax.mail:1.6.2'
implementation libs.mail
implementation libs.jackson
implementation libs.gson
implementation libs.json
implementation 'org.hibernate.orm:hibernate-community-dialects'
testImplementation('org.springframework.boot:spring-boot-starter-test') {
exclude group: 'org.junit.vintage', module: 'junit-vintage-engine'
+8
View File
@@ -20,6 +20,10 @@ springboot = "3.2.5"
springdependencies = "1.1.4"
vaadin = "24.3.8"
lombok = "8.6"
gson = "2.9.0"
jackson = "2.16.1"
json_simple = "1.1.1"
mail = "1.6.2"
[libraries]
args4j = { module = "args4j:args4j", version.ref = "args4j" }
@@ -31,6 +35,10 @@ mockito = { module = "org.mockito:mockito-all", version.ref = "mockito" }
picocli = { module = "info.picocli:picocli", version.ref = "picoli" }
slf4j = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
hsqldb = { module = "org.hsqldb:hsqldb", version.ref = "hsqldb" }
gson = { module = "com.google.code.gson:gson", version.ref = "gson" }
jackson = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" }
json = { module = "com.googlecode.json-simple:json-simple", version.ref ="json_simple" }
mail = { module = "com.sun.mail:javax.mail", version.ref ="mail" }
sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite" }
vaadin-bom = { module = "com.vaadin:vaadin-bom", version.ref = "vaadin" }
asciidoctorGradleJvmGems = { module = "org.asciidoctor:asciidoctor-gradle-jvm-gems", version.ref= "asciidoctor" }
@@ -1,7 +1,9 @@
package de.thpeetz.kontor.admin;
import com.vaadin.flow.component.page.Meta;
import de.thpeetz.kontor.admin.data.*;
import de.thpeetz.kontor.admin.repository.AuthorizationMatrixRepository;
import de.thpeetz.kontor.admin.repository.MailAccountRepository;
import de.thpeetz.kontor.admin.repository.UserRepository;
import de.thpeetz.kontor.admin.services.AdminService;
import de.thpeetz.kontor.admin.services.MetaDataService;
import de.thpeetz.kontor.mailclient.data.MailAccount;
@@ -5,10 +5,15 @@ import jakarta.persistence.*;
import jakarta.validation.constraints.NotNull;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
@Slf4j
@Entity
@Getter
@Setter
@@ -23,4 +28,11 @@ public class MetaDataTable extends AbstractEntity {
@OneToMany(fetch = FetchType.EAGER, mappedBy = "table")
private List<MetaDataColumn> tableColumns = new LinkedList<>();
public void updateTableName(String value) {
if (!this.getTableName().equals(value)) {
this.setTableName(value);
log.info("update tableName");
}
}
}
@@ -1,8 +1,10 @@
package de.thpeetz.kontor.admin.data;
package de.thpeetz.kontor.admin.repository;
import java.util.List;
import java.util.UUID;
import de.thpeetz.kontor.admin.data.AuthorizationMatrix;
import de.thpeetz.kontor.admin.data.Role;
import de.thpeetz.kontor.admin.data.User;
import org.springframework.data.jpa.repository.JpaRepository;
public interface AuthorizationMatrixRepository extends JpaRepository<AuthorizationMatrix, String> {
@@ -1,4 +1,4 @@
package de.thpeetz.kontor.admin.data;
package de.thpeetz.kontor.admin.repository;
import de.thpeetz.kontor.mailclient.data.MailAccount;
import org.springframework.data.jpa.repository.JpaRepository;
@@ -1,5 +1,7 @@
package de.thpeetz.kontor.admin.data;
package de.thpeetz.kontor.admin.repository;
import de.thpeetz.kontor.admin.data.MetaDataColumn;
import de.thpeetz.kontor.admin.data.MetaDataTable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@@ -1,5 +1,6 @@
package de.thpeetz.kontor.admin.data;
package de.thpeetz.kontor.admin.repository;
import de.thpeetz.kontor.admin.data.MetaDataTable;
import org.springframework.data.jpa.repository.JpaRepository;
public interface MetaDataTableRepository extends JpaRepository<MetaDataTable, String> {
@@ -1,5 +1,6 @@
package de.thpeetz.kontor.admin.data;
package de.thpeetz.kontor.admin.repository;
import de.thpeetz.kontor.admin.data.ModuleData;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@@ -1,7 +1,8 @@
package de.thpeetz.kontor.admin.data;
package de.thpeetz.kontor.admin.repository;
import java.util.List;
import de.thpeetz.kontor.admin.data.Role;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@@ -1,7 +1,8 @@
package de.thpeetz.kontor.admin.data;
package de.thpeetz.kontor.admin.repository;
import java.util.List;
import de.thpeetz.kontor.admin.data.User;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
@@ -6,11 +6,11 @@ import java.util.List;
import org.springframework.stereotype.Service;
import de.thpeetz.kontor.admin.data.AuthorizationMatrix;
import de.thpeetz.kontor.admin.data.AuthorizationMatrixRepository;
import de.thpeetz.kontor.admin.repository.AuthorizationMatrixRepository;
import de.thpeetz.kontor.admin.data.Role;
import de.thpeetz.kontor.admin.data.RoleRepository;
import de.thpeetz.kontor.admin.repository.RoleRepository;
import de.thpeetz.kontor.admin.data.User;
import de.thpeetz.kontor.admin.data.UserRepository;
import de.thpeetz.kontor.admin.repository.UserRepository;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -1,6 +1,9 @@
package de.thpeetz.kontor.admin.services;
import de.thpeetz.kontor.admin.data.*;
import de.thpeetz.kontor.admin.repository.AuthorizationMatrixRepository;
import de.thpeetz.kontor.admin.repository.RoleRepository;
import de.thpeetz.kontor.admin.repository.UserRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.GrantedAuthority;
@@ -1,7 +1,7 @@
package de.thpeetz.kontor.admin.services;
import de.thpeetz.kontor.mailclient.data.MailAccount;
import de.thpeetz.kontor.admin.data.MailAccountRepository;
import de.thpeetz.kontor.admin.repository.MailAccountRepository;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@@ -3,12 +3,16 @@ package de.thpeetz.kontor.admin.services;
import org.springframework.stereotype.Service;
import de.thpeetz.kontor.admin.data.MetaDataColumn;
import de.thpeetz.kontor.admin.data.MetaDataColumnRepository;
import de.thpeetz.kontor.admin.repository.MetaDataColumnRepository;
import de.thpeetz.kontor.admin.data.MetaDataTable;
import de.thpeetz.kontor.admin.data.MetaDataTableRepository;
import de.thpeetz.kontor.admin.repository.MetaDataTableRepository;
import lombok.extern.slf4j.Slf4j;
import java.sql.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
@Service
@@ -23,6 +27,10 @@ public class MetaDataService {
this.metaDataColumnRepository = metaDataColumnRepository;
}
public List<MetaDataTable> findAllTables() {
return metaDataTableRepository.findAll();
}
public MetaDataTable getTable(String tableName) {
MetaDataTable table = metaDataTableRepository.findByTableName(tableName);
if (table == null) {
@@ -34,6 +42,14 @@ public class MetaDataService {
return table;
}
private void deleteTable(MetaDataTable metaDataTable) {
List<MetaDataColumn> columns = metaDataTable.getTableColumns();
for (MetaDataColumn column: columns) {
metaDataColumnRepository.delete(column);
}
metaDataTableRepository.delete(metaDataTable);
}
public void getColumn(MetaDataTable table, String columnName, String columnSyncName, String columnType, String columnModifier, Integer columnOrder, Boolean isShown, String columnLabel, Boolean showFilter, String filterLabel) {
this.getColumn(table, columnName, columnSyncName, columnType, columnModifier, columnOrder, isShown, columnLabel, showFilter, filterLabel, null);
}
@@ -123,7 +139,46 @@ public class MetaDataService {
metaDataColumnRepository.save(metaDataColumn);
}
public List<MetaDataTable> findAllTables() {
return metaDataTableRepository.findAll();
public String importData(Map<String, String> fields) {
AtomicReference<String> status = new AtomicReference<>("unknown");
String id = fields.get("id");
Optional<MetaDataTable> optional = metaDataTableRepository.findById(id);
if (optional.isEmpty()) {
log.info(" not found: {} with {}", id, fields);
status.set(id + "not found");
MetaDataTable checkExisting = metaDataTableRepository.findByTableName(fields.get("table_name"));
if (checkExisting != null) {
log.info("entry already there with different id ({}), will be deleted", checkExisting.getId());
deleteTable(checkExisting);
}
MetaDataTable metaDataTable = new MetaDataTable();
metaDataTable.setId(id);
metaDataTable.setTableName(fields.get("table_name"));
metaDataTableRepository.save(metaDataTable);
} else {
optional.ifPresent( entry -> {
log.info(" found: {}", entry.getTableName());
updateFields(entry, fields);
metaDataTableRepository.save(entry);
status.set("found");
});
}
return status.get();
}
private void updateFields(MetaDataTable metaDataTable, Map<String, String> fields) {
for (Map.Entry<String, String> entry : fields.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
switch (key) {
case "id", "created_date", "last_modified_date", "version":
break;
case "table_name":
metaDataTable.updateTableName(value);
break;
default:
log.info("field {} is unknown for table {}", key, MetaDataTable.class.getName());
}
}
}
}
@@ -1,7 +1,7 @@
package de.thpeetz.kontor.admin.services;
import de.thpeetz.kontor.admin.data.ModuleData;
import de.thpeetz.kontor.admin.data.ModuleDataRepository;
import de.thpeetz.kontor.admin.repository.ModuleDataRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -31,4 +31,15 @@ public abstract class AbstractEntity {
@LastModifiedDate
private Date lastModifiedDate;
public void updateVersion(String value) {
if (value == null) {
log.info("value not given");
return;
}
if (version != Integer.valueOf(value).intValue()) {
log.info("update version");
version = Integer.valueOf(value).intValue();
}
}
}
@@ -4,6 +4,7 @@ import com.vaadin.flow.component.applayout.AppLayout;
import com.vaadin.flow.component.applayout.DrawerToggle;
import com.vaadin.flow.component.html.H1;
import com.vaadin.flow.component.html.H2;
import com.vaadin.flow.component.icon.VaadinIcon;
import com.vaadin.flow.component.orderedlayout.FlexComponent;
import com.vaadin.flow.component.orderedlayout.HorizontalLayout;
import com.vaadin.flow.component.orderedlayout.Scroller;
@@ -17,15 +18,14 @@ import de.thpeetz.kontor.admin.AdminConstants;
import de.thpeetz.kontor.admin.services.AdminService;
import de.thpeetz.kontor.bookshelf.BookshelfConstants;
import de.thpeetz.kontor.comics.ComicConstants;
import de.thpeetz.kontor.data.views.DataManagementView;
import de.thpeetz.kontor.mailclient.views.EmailView;
import de.thpeetz.kontor.media.MediaConstants;
import de.thpeetz.kontor.security.SecurityService;
import de.thpeetz.kontor.tysc.TyscConstants;
import lombok.extern.slf4j.Slf4j;
import org.springframework.security.core.GrantedAuthority;
import java.util.ArrayList;
import java.util.Collection;
@Slf4j
public class MainLayout extends AppLayout {
@@ -87,6 +87,7 @@ public class MainLayout extends AppLayout {
sideNav.addItem(BookshelfConstants.getBookshelfNavigation());
sideNav.addItem(MediaConstants.getMediaNavigation(roles));
sideNav.addItem(new SideNavItem("Emails", EmailView.class));
sideNav.addItem(new SideNavItem("Data Management", DataManagementView.class, VaadinIcon.DATABASE.create()));
securityService.getAuthenticatedUser().ifPresent(user -> {
log.info("User {} found", user.getUsername());
boolean isAdmin = user.getAuthorities().stream()
@@ -0,0 +1,40 @@
package de.thpeetz.kontor.data.services;
import de.thpeetz.kontor.admin.data.MetaDataTable;
import de.thpeetz.kontor.admin.repository.MetaDataTableRepository;
import de.thpeetz.kontor.admin.services.MetaDataService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
@Service
public class DataManagementService {
@Autowired
MetaDataTableRepository metaDataTableRepository;
@Autowired
MetaDataService metaDataService;
public DataManagementService() {
}
public String getEntry(String nodeName, Map<String, String> fields) {
AtomicReference<String> status = new AtomicReference<>("unknown");
switch (nodeName) {
case "meta_data_table":
//status.set(metaDataService.importData(fields));
break;
default:
log.debug("import for {} not implemented", nodeName);
break;
}
return status.get();
}
}
@@ -0,0 +1,49 @@
package de.thpeetz.kontor.data.views;
import com.vaadin.flow.component.checkbox.Checkbox;
import com.vaadin.flow.component.orderedlayout.VerticalLayout;
import com.vaadin.flow.router.PageTitle;
import com.vaadin.flow.router.Route;
import com.vaadin.flow.spring.annotation.SpringComponent;
import de.thpeetz.kontor.common.views.MainLayout;
import de.thpeetz.kontor.data.services.DataManagementService;
import jakarta.annotation.security.PermitAll;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Scope;
import java.io.File;
@Slf4j
@SpringComponent
@Scope("prototype")
@PermitAll
@Route(value = "common/import", layout = MainLayout.class)
@PageTitle("Import Data | Kontor")
public class DataManagementView extends VerticalLayout {
DataManagementService service;
public DataManagementView(DataManagementService service) {
this.service = service;
File uploadFolder = getUploadFolder();
UploadArea uploadArea = new UploadArea(uploadFolder);
Checkbox pruneTables = new Checkbox("Prune Table");
ImportArea importArea = new ImportArea(uploadFolder, pruneTables, service);
uploadArea.getUploadField().addSucceededListener(e -> {
uploadArea.hideErrorField();
importArea.processFiles();
uploadArea.getUploadField().clearFileList();
});
add(uploadArea, pruneTables, importArea);
}
private File getUploadFolder() {
File folder = new File("uploaded-files");
if (!folder.exists()) {
folder.mkdirs();
}
return folder;
}
}
@@ -0,0 +1,111 @@
package de.thpeetz.kontor.data.views;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.vaadin.flow.component.checkbox.Checkbox;
import com.vaadin.flow.component.html.H4;
import com.vaadin.flow.component.html.Paragraph;
import com.vaadin.flow.component.orderedlayout.VerticalLayout;
import de.thpeetz.kontor.data.services.DataManagementService;
import lombok.extern.slf4j.Slf4j;
import org.json.simple.parser.ParseException;
import java.io.*;
import java.util.*;
@Slf4j
public class ImportArea extends VerticalLayout {
protected File uploadFolder;
DataManagementService service;
boolean pruneTables = false;
public ImportArea(File uploadFolder, Checkbox pruneTablesCheckbox, DataManagementService service) {
this.uploadFolder = uploadFolder;
this.service = service;
this.pruneTables = pruneTablesCheckbox.isEnabled();
setMargin(true);
}
public void processFiles() {
removeAll();
if (pruneTables) {
add(new Paragraph("prune all tables"));
}
add(new H4("Process file"));
for (File file: uploadFolder.listFiles()) {
add(new Paragraph("reading " + file.getName()));
try {
processFile(file);
} catch (ParseException e) {
throw new RuntimeException(e);
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
private void processFile(File file) throws FileNotFoundException, IOException, ParseException {
ObjectMapper objectMapper = new ObjectMapper();
JsonNode node = objectMapper.readTree(file);
log.info("parsing: {}, {}", node.getNodeType(), node.size());
Iterator<String> fieldNamesIterator = node.fieldNames();
while (fieldNamesIterator.hasNext()) {
String fieldName = fieldNamesIterator.next();
parseRoot(node, fieldName);
}
}
private void parseRoot(JsonNode parentNode, String field) {
JsonNode node = parentNode.get(field);
String nodeType = String.valueOf(node.getNodeType());
if (nodeType.equals("ARRAY")) {
log.info(" {}: {}", field, node.size());
Iterator<JsonNode> elements = node.elements();
while (elements.hasNext()) {
JsonNode subNode = elements.next();
String status = parseNode(subNode, field);
add(new Paragraph(status));
}
} else {
log.info("unknown type: {}", nodeType);
}
}
private String parseNode(JsonNode node, String nodeName) {
Map<String, String> fields = parseFields(node);
return service.getEntry(nodeName, fields);
}
private Map<String, String> parseFields(JsonNode node) {
Map<String, String> fields = new HashMap<>();
node.fieldNames().forEachRemaining(field -> {
JsonNode value = node.get(field);
log.debug("type: {}", value.getNodeType());
JsonNodeType nodeType = value.getNodeType();
switch (nodeType) {
case NUMBER:
fields.put(field, String.valueOf(value.intValue()));
break;
case STRING:
String stringValue = value.textValue();
if (stringValue.contains("\"")) {
log.debug("remove double quotes");
String stripped = stringValue.substring(1, stringValue.length()-1);
fields.put(field, stripped);
}
fields.put(field, value.textValue());
break;
default:
log.debug("no implementation for type {}", nodeType.name());
}
});
log.debug("fields: {}", fields);
return fields;
}
}
@@ -0,0 +1,58 @@
package de.thpeetz.kontor.data.views;
import com.vaadin.flow.component.html.Span;
import com.vaadin.flow.component.orderedlayout.VerticalLayout;
import com.vaadin.flow.component.upload.MultiFileReceiver;
import com.vaadin.flow.component.upload.Receiver;
import com.vaadin.flow.component.upload.Upload;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@Slf4j
public class UploadArea extends VerticalLayout {
@Getter
private final Upload uploadField;
private final Span errorField;
public UploadArea(File uploadFolder) {
uploadField = new Upload(createFileReceiver(uploadFolder));
uploadField.setMaxFiles(1);
uploadField.setMaxFileSize(2*1024*1024);
errorField = new Span();
errorField.setVisible(false);
errorField.getStyle().set("color", "red");
uploadField.addFailedListener(e -> showErrorMessage(e.getReason().getMessage()));
uploadField.addFileRejectedListener(e -> showErrorMessage(e.getErrorMessage()));
add(uploadField, errorField);
}
public void hideErrorField() {
errorField.setVisible(false);
}
private Receiver createFileReceiver(File uploadFolder) {
return (MultiFileReceiver) (filename, mimetype) -> {
File file = new File(uploadFolder, filename);
try {
return new FileOutputStream(file);
}
catch (FileNotFoundException fnfe) {
log.info(fnfe.getStackTrace().toString());
return null;
}
};
}
private void showErrorMessage(String message) {
errorField.setVisible(true);
errorField.setText(message);
}
}
@@ -17,6 +17,10 @@ spring:
mode: never
mustache:
check-template-location: false
servlet:
multipart:
max-file-size: 10MB
max-request-size: 10MB
management:
endpoints:
web: