add missing endpoints for creating items

kontor-api/src/apis/base.py

@@ -16,9 +16,9 @@ from src.apis.version1.comics import (
     issuework,
 )
 from src.apis.version1.media import (
-    mediaactor,
+    actor,
+    file,
     mediaactorfile,
-    mediafile,
     mediavideo,
     mediaarticle,
 )
@@ -93,7 +93,7 @@ api_router.include_router(
     dependencies=[Depends(get_current_user_from_token)],
 )
 api_router.include_router(
-    mediafile.router,
+    file.router,
     prefix="/media",
     tags=["media"],
     dependencies=[Depends(get_current_user_from_token)],
@@ -111,7 +111,7 @@ api_router.include_router(
     dependencies=[Depends(get_current_user_from_token)],
 )
 api_router.include_router(
-    mediaactor.router,
+    actor.router,
     prefix="/media",
     tags=["media"],
     dependencies=[Depends(get_current_user_from_token)],

kontor-api/src/apis/version1/media/actor.py

@@ -2,7 +2,7 @@ from typing import List
 
 from fastapi import APIRouter, status, HTTPException
 from src.core.log_conf import logger
-from src.db.repository.media import create_new_mediaactor, delete_mediaactor
+from src.db.repository.media.actor import delete_mediaactor, import_mediaactor
 from src.db.session import SessionDep
 from src.schema.media.actor import MediaActorModel, MediaActorResponse, actor_to_response
 from src.db.models.media import MediaActor
@@ -38,8 +38,8 @@ def delete_actor(actor_id: str, db: SessionDep):
 def add_actor(new_actor: MediaActorModel, db: SessionDep) -> MediaActorResponse:
     logger.info(f"add actor {new_actor.url}")
     try:
-        mediaActor: MediaActor = create_new_mediaactor(new_actor, db)
+        mediaActor: MediaActor = import_mediaactor(db, new_actor)
-    except:
+    except Exception as exception:
-        raise HTTPException(status_code=409, detail="Link duplicate")
+        raise HTTPException(status_code=409, detail=f"Link duplicate: {exception}")
     response = actor_to_response(mediaActor)
     return response

kontor-api/src/apis/version1/media/file.py

@@ -2,17 +2,14 @@ from typing import List
 
 from fastapi import APIRouter, status, HTTPException
 from src.core.log_conf import logger
-from src.db.repository.media import (
+from src.db.repository.media.actorfile import create_new_mediaactorfile
-    create_new_mediaactorfile,
+from src.db.repository.media.file import delete_mediafile, import_mediafile
-    create_new_mediafile,
-    delete_mediafile,
-)
 from src.db.session import SessionDep
 from src.schema.media.actor import MediaActorResponse, actor_to_response
 from src.schema.media.actorfile import MediaActorFileResponse, actorfile_to_response
 from src.schema.media.file import (
+    MediaFileModel,
     MediaFileResponse,
-    Link,
     file_to_response,
     file_to_model,
 )
@@ -150,11 +147,11 @@ def update_file(
 
 
 @router.post("/files", status_code=status.HTTP_201_CREATED)
-def add_file(new_link: Link, db: SessionDep) -> MediaFileResponse:  # type: ignore
+def add_file(new_file: MediaFileModel, db: SessionDep) -> MediaFileResponse:
-    logger.info("add url %s", new_link.url)
+    logger.info("add mediafile %s", new_file)
     try:
-        mediaFile: MediaFile = create_new_mediafile(new_link.url, db)
+        mediaFile: MediaFile = import_mediafile(db, new_file)
     except:
-        raise HTTPException(status_code=409, detail="Link duplicate")
+        raise HTTPException(status_code=409, detail="MediaFile duplicate")
     response = file_to_response(mediaFile)
     return response

kontor-api/src/apis/version1/media/mediaactorfile.py

@@ -1,7 +1,7 @@
 from typing import List
 from fastapi import APIRouter, status, HTTPException
 from src.db.models.media import MediaActorFile
-from src.db.repository.media import delete_mediaactorfile
+from src.db.repository.media.actorfile import delete_mediaactorfile
 from src.db.session import SessionDep
 from src.schema.media.actorfile import MediaActorFileResponse, actorfile_to_response
 

kontor-api/src/db/repository/media.py

@@ -1,93 +0,0 @@
-from sqlalchemy.orm import Session
-import uuid
-from datetime import datetime
-from src.core.log_conf import logger
-from src.db.models.media import MediaActor, MediaActorFile, MediaFile, MediaVideo
-from src.schema.media.actor import MediaActorModel
-from src.webapps.media.forms import AddLinkForm
-
-
-def create_new_video(video: AddLinkForm, db: Session) -> MediaVideo:
-    print(video.url)
-    media_video = MediaVideo()
-    media_video.id = str(uuid.uuid4())
-    media_video.url = str(video.url)
-    media_video.created_date = datetime.now()
-    media_video.last_modified_date = datetime.now()
-    media_video.review = True
-    media_video.should_download = True
-    db.add(media_video)
-    db.commit()
-    db.refresh(media_video)
-    print(media_video)
-    return media_video
-
-def create_new_mediafile(link: str, db: Session) -> MediaFile:
-    logger.info("create MediaFile with url {link}")
-    media_file: MediaFile = MediaFile()
-    media_file.id = str(uuid.uuid4())
-    media_file.url = link
-    media_file.created_date = datetime.now()
-    media_file.last_modified_date = datetime.now()
-    media_file.version = 0
-    media_file.review = True
-    media_file.should_download = True
-    db.add(media_file)
-    db.commit()
-    db.refresh(media_file)
-    logger.info(f"created {media_file}")
-    return media_file
-
-def delete_mediafile(db: Session, media_file_id: str):
-    logger.info(f"delete MediaFile with id {media_file_id}")
-    media_file = db.get(MediaFile, media_file_id)
-    db.delete(media_file)
-    db.commit()
-
-def create_new_mediaactor(new_actor: MediaActorModel, db: Session) -> MediaActor:
-    logger.info(f"create MediaActor with url {new_actor.url}")
-    media_actor: MediaActor = MediaActor()
-    media_actor.id = str(uuid.uuid4())
-    if new_actor.name is not None:
-        media_actor.name = new_actor.name
-    media_actor.url = new_actor.url
-    media_actor.created_date = datetime.now()
-    media_actor.last_modified_date = datetime.now()
-    media_actor.version = 0
-    db.add(media_actor)
-    db.commit()
-    db.refresh(media_actor)
-    logger.info(f"created {media_actor}")
-    return media_actor
-
-def delete_mediaactor(db: Session, actor_id: str):
-    logger.info(f"delete MediaActor with id {actor_id}")
-    media_actor = db.get(MediaActor, actor_id)
-    if media_actor is not None:
-        actor_files = media_actor.media_actor_files
-        for actor_file in actor_files:
-            delete_mediaactorfile(db, actorfile_id=actor_file.id)
-    db.refresh(media_actor)
-    db.delete(media_actor)
-    db.commit()
-
-def create_new_mediaactorfile(db: Session, actor_id: str, file_id: str) -> MediaActorFile:
-    logger.info(f"create MediaActorFile with actor {actor_id} and file {file_id}")
-    media_actor_file: MediaActorFile = MediaActorFile()
-    media_actor_file.id = str(uuid.uuid4())
-    media_actor_file.created_date = datetime.now()
-    media_actor_file.last_modified_date = datetime.now()
-    media_actor_file.version = 0
-    media_actor_file.media_actor_id = actor_id
-    media_actor_file.media_file_id = file_id
-    db.add(media_actor_file)
-    db.commit()
-    db.refresh(media_actor_file)
-    return media_actor_file
-
-def delete_mediaactorfile(db: Session, actorfile_id: str):
-    logger.info(f"delete MediaActorFile with id {actorfile_id}")
-    media_actorfile = db.get(MediaActorFile, actorfile_id)
-    db.delete(media_actorfile)
-    db.commit()
-        

kontor-api/src/db/repository/media/actor.py

@@ -0,0 +1,66 @@
+from datetime import datetime
+import uuid
+
+from sqlalchemy.orm import Session
+
+from src.core.log_conf import logger
+from src.db.models.media import MediaActor
+from src.db.repository.media.actorfile import delete_mediaactorfile
+from src.schema.media.actor import MediaActorModel
+
+
+def create_new_mediaactor(new_actor: MediaActorModel, db: Session) -> MediaActor:
+    logger.info(f"create MediaActor with url {new_actor.url}")
+    media_actor: MediaActor = MediaActor()
+    media_actor.id = str(uuid.uuid4())
+    if new_actor.name is not None:
+        media_actor.name = new_actor.name
+    media_actor.url = new_actor.url
+    media_actor.created_date = datetime.now()
+    media_actor.last_modified_date = datetime.now()
+    media_actor.version = 0
+    db.add(media_actor)
+    db.commit()
+    db.refresh(media_actor)
+    logger.info(f"created {media_actor}")
+    return media_actor
+
+def delete_mediaactor(db: Session, actor_id: str):
+    logger.info(f"delete MediaActor with id {actor_id}")
+    media_actor = db.get(MediaActor, actor_id)
+    if media_actor is not None:
+        actor_files = media_actor.media_actor_files
+        for actor_file in actor_files:
+            delete_mediaactorfile(db, actorfile_id=actor_file.id)
+    db.refresh(media_actor)
+    db.delete(media_actor)
+    db.commit()
+
+def import_mediaactor(db: Session, new_actor: MediaActorModel) -> MediaActor:
+    """
+    import MediaFile and set missing values with default ones.
+    """
+    logger.info("import MediaActor with %s", new_actor)
+    media_actor: MediaActor = MediaActor()
+    media_actor.id = new_actor.id
+    if new_actor.created_date:
+        media_actor.created_date = new_actor.created_date
+    else:
+        media_actor.created_date = datetime.now()
+    if new_actor.last_modified_date:
+        media_actor.last_modified_date = new_actor.last_modified_date
+    else:
+        media_actor.last_modified_date = datetime.now()
+    media_actor.version = new_actor.version
+    if new_actor.name:
+        media_actor.name = new_actor.name
+    else:
+        media_actor.name = ""
+    if new_actor.url:
+        media_actor.url = new_actor.url
+    else:
+        media_actor.url = ""
+    db.add(media_actor)
+    db.commit()
+    db.refresh(media_actor)
+    return media_actor

kontor-api/src/db/repository/media/actorfile.py

@@ -0,0 +1,39 @@
+from datetime import datetime
+import uuid
+
+from sqlalchemy.orm import Session
+
+from src.core.log_conf import logger
+from src.db.models.media import MediaActorFile
+from src.schema.media.actor import MediaActorModel
+from src.schema.media.actorfile import MediaActorFileModel
+
+
+def create_new_mediaactorfile(db: Session, actor_id: str, file_id: str) -> MediaActorFile:
+    logger.info(f"create MediaActorFile with actor {actor_id} and file {file_id}")
+    media_actor_file: MediaActorFile = MediaActorFile()
+    media_actor_file.id = str(uuid.uuid4())
+    media_actor_file.created_date = datetime.now()
+    media_actor_file.last_modified_date = datetime.now()
+    media_actor_file.version = 0
+    media_actor_file.media_actor_id = actor_id
+    media_actor_file.media_file_id = file_id
+    db.add(media_actor_file)
+    db.commit()
+    db.refresh(media_actor_file)
+    return media_actor_file
+
+def delete_mediaactorfile(db: Session, actorfile_id: str):
+    logger.info(f"delete MediaActorFile with id {actorfile_id}")
+    media_actorfile = db.get(MediaActorFile, actorfile_id)
+    db.delete(media_actorfile)
+    db.commit()
+
+def import_mediaactorfile(db: Session, new_actorfile: MediaActorFileModel) -> MediaActorFile:
+    """
+    Import MediaFile and set missing values with default ones.
+    """
+    logger.info("import MediaActorFile with %s", new_actorfile)
+    media_actor_file: MediaActorFile = MediaActorFile()
+    return media_actor_file
+    

kontor-api/src/db/repository/media/file.py

@@ -0,0 +1,70 @@
+from datetime import datetime
+import uuid
+
+from sqlalchemy.orm import Session
+
+from src.core.log_conf import logger
+from src.db.models.media import MediaFile
+from src.schema.media.file import MediaFileModel
+
+
+def create_new_mediafile(link: str, db: Session) -> MediaFile:
+    logger.info("create MediaFile with url {link}")
+    media_file: MediaFile = MediaFile()
+    media_file.id = str(uuid.uuid4())
+    media_file.url = link
+    media_file.created_date = datetime.now()
+    media_file.last_modified_date = datetime.now()
+    media_file.version = 0
+    media_file.review = True
+    media_file.should_download = True
+    db.add(media_file)
+    db.commit()
+    db.refresh(media_file)
+    logger.info(f"created {media_file}")
+    return media_file
+
+def delete_mediafile(db: Session, media_file_id: str):
+    logger.info(f"delete MediaFile with id {media_file_id}")
+    media_file = db.get(MediaFile, media_file_id)
+    db.delete(media_file)
+    db.commit()
+
+def import_mediafile(db: Session, new_file: MediaFileModel) -> MediaFile:
+    """
+    import MediaActor and set missing values with defautl ones.
+    """
+    logger.info("import MediaFile with %s", new_file)
+    media_file: MediaFile = MediaFile()
+    media_file.id = new_file.id
+    if new_file.created_date:
+        media_file.created_date = new_file.created_date
+    else:
+        media_file.created_date = datetime.now()
+    if new_file.last_modified_date:
+        media_file.last_modified_date = new_file.last_modified_date
+    else:
+        media_file.last_modified_date = datetime.now()
+    media_file.version = new_file.version
+    if new_file.title:
+        media_file.title = new_file.title
+    else:
+        media_file.title = ""
+    if new_file.file_name:
+        media_file.file_name = new_file.file_name
+    else:
+        media_file.file_name = ""
+    if new_file.cloud_link:
+        media_file.cloud_link = new_file.cloud_link
+    else:
+        media_file.cloud_link = ""
+    if new_file.url:
+        media_file.url = new_file.url
+    else:
+        media_file.url = ""
+    media_file.review = new_file.review
+    media_file.should_download = new_file.should_download
+    db.add(media_file)
+    db.commit()
+    db.refresh(media_file)
+    return media_file

kontor-api/src/db/repository/media/video.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+import uuid
+
+from sqlalchemy.orm import Session
+
+from src.db.models.media import MediaVideo
+from src.webapps.media.forms import AddLinkForm
+
+
+def create_new_video(video: AddLinkForm, db: Session) -> MediaVideo:
+    print(video.url)
+    media_video = MediaVideo()
+    media_video.id = str(uuid.uuid4())
+    media_video.url = str(video.url)
+    media_video.created_date = datetime.now()
+    media_video.last_modified_date = datetime.now()
+    media_video.review = True
+    media_video.should_download = True
+    db.add(media_video)
+    db.commit()
+    db.refresh(media_video)
+    print(media_video)
+    return media_video

kontor-api/src/schema/media/actor.py

@@ -26,5 +26,9 @@ def actor_to_response(actor: MediaActor) -> MediaActorResponse:
 
 
 class MediaActorModel(BaseModel):
+    id: str
+    created_date: datetime
+    last_modified_date: datetime
+    version: int
     name: Optional[str]
-    url: str
+    url: Optional[str]

kontor-api/src/schema/media/actorfile.py

@@ -23,3 +23,13 @@ def actorfile_to_response(actorfile:  MediaActorFile) -> MediaActorFileResponse:
             media_file_id=actorfile.media_file_id
     )
     return response
+
+
+class MediaActorFileModel(BaseModel):
+    id: str
+    created_date: datetime
+    last_modified_date: datetime
+    version: int
+    media_actor_id: str
+    media_file_id: Optional[str]
+    

kontor-api/src/schema/media/file.py

@@ -57,6 +57,22 @@ def file_to_model(model: MediaFileResponse, mediafile: MediaFile) -> MediaFile:
     return mediafile
 
 
+class MediaFileModel(BaseModel):
+    """
+    Pydantic model to import MediaFile.
+    """
+    id: str
+    created_date: Optional[datetime]
+    last_modified_date: Optional[datetime]
+    version: int = 0
+    title: Optional[str]
+    file_name: Optional[str]
+    cloud_link: Optional[str]
+    url: Optional[str]
+    review: bool = True
+    should_download: bool = True
+    
+
 class Link(BaseModel):
     """
     PYdantic model for uploading url.

kontor-api/src/webapps/media/forms.py

@@ -10,7 +10,7 @@ class AddLinkForm:
 
     async def load_data(self):
         form = await self.request.form()
-        self.url = form.get("url")
+        self.url = str(form.get("url"))
 
     def is_valid(self):
         if not self.url or not (self.url.__contains__("http")):

kontor-api/src/webapps/media/route_videos.py

@@ -4,10 +4,9 @@ from fastapi.templating import Jinja2Templates
 
 from src.core.security import get_current_user_from_token
 from src.db.models.media import MediaVideo
-from src.db.repository.media import create_new_video
 from src.db.models.admin import Profile
+from src.db.repository.media.video import create_new_video
 from src.db.session import SessionDep
-from src.schema.media.video import AddLink
 from src.webapps.media.forms import AddLinkForm
 
 templates = Jinja2Templates(directory="src/templates")
@@ -40,8 +39,8 @@ async def post_video_link(request: Request, db: SessionDep):
     await form.load_data()
     if form.is_valid():
         try:
-            video = AddLink(**form.__dict__)
+            #video = AddLink(**form.__dict__)
-            mediavideo = create_new_video(video=video, db=db)
+            mediavideo = create_new_video(video=form, db=db)
             return responses.RedirectResponse(f"media/videos/{mediavideo.id}", status_code=status.HTTP_302_FOUND)
         except Exception as e:
             print(e)

kontor-scripts/api.py

@@ -78,6 +78,12 @@ class Option:
             return f"/{self.value}"
 
 
+class CredentialsValidationException(Exception):
+    """
+    Raised when login failed or token is outdated.
+    """
+
+    
 class EndPointNotAvailableException(Exception):
     """
     Raised when calling an not existing endpoint.
@@ -104,30 +110,38 @@ class Server:
     url: str
     token: str
     token_type: str
+    email: str
+    password: str
     timeout: int
 
-    def login(self, login: Login, log: Logger):
+    def login(self, log: Logger, refresh_token: bool= False):
         """
         get token from server by calling login endpoint.
         """
+        log.debug("call login and retrieve token")
         if not self.token:
-            log.info("Call login first")
+            self.__get_token__(log=log)
-            login_url = f"{self.url}/login"
+        if refresh_token:
-            login_data = {}
+            self.__get_token__(log=log)
-            login_data["email"] = login.email
+
-            login_data["password"] = login.password
+    def __get_token__(self, log: Logger):
-            response = requests.post(login_url, json=login_data, timeout=self.timeout)
+        log.info("Call login first")
-            status = response.status_code
+        login_url = f"{self.url}/login"
-            log.info(f"Status: {status}")
+        login_data = {}
-            if status != 200:
+        login_data["email"] = self.email
-                log.fatal("authentication failed")
+        login_data["password"] = self.password
-                return
+        response = requests.post(login_url, json=login_data, timeout=self.timeout)
-            data = response.json()
+        status = response.status_code
-            log.debug(f"got data: {data}")
+        log.info(f"Status: {status}")
-            token = data["access_token"]
+        if status != 200:
-            token_type = data["token_type"]
+            log.fatal("authentication failed")
-            self.token = str(token)
+            return
-            self.token_type = str(token_type)
+        data = response.json()
+        log.debug(f"got data: {data}")
+        token = data["access_token"]
+        token_type = data["token_type"]
+        self.token = str(token)
+        self.token_type = str(token_type)
 
     def request(self, log: Logger, table: str, param: Optional[Option] = None):
         """
@@ -140,6 +154,10 @@ class Server:
         headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"}
         response = requests.get(url, headers=headers, timeout=self.timeout)
         log.debug(f"Status: {response.status_code}")
+        if response.status_code == 401:
+            self.login(log, refresh_token=True)
+            headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"}
+            response = requests.get(url, headers=headers, timeout=self.timeout)
         if response.status_code == 404:
             raise EndPointNotAvailableException
         data = response.json()
@@ -159,6 +177,22 @@ class Server:
             raise EndPointNotAvailableException
         data = update.json()
         return data
+    
+    def create(self, log: Logger, table: str, new_item: dict):
+        """
+        Create item in Kontor-API instance.
+        """
+        url: str = f"{self.url}/{MAPPING[table]}"
+        headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"}
+        create = requests.post(url, headers=headers, json=new_item, timeout=self.timeout)
+        log.info(f"Status: {create.status_code}")
+        if create.status_code == 404:
+            raise EndPointNotAvailableException
+        if create.status_code == 409:
+            log.fatal("Create Exception %s", create.json())
+        data = create.json()
+        return data
+        
 
 
 @dataclass
@@ -167,7 +201,6 @@ class ApiConfig:
     Dataclass to define required contents of configuration file.
     """
 
-    login: Login
     server: List[Server]
 
     def get_server(self, server_name: str) -> Optional[Server]:
@@ -199,13 +232,17 @@ def get_logger(level, config: str):
                 logging.getLogger("requests").setLevel(logging.WARNING)
                 logging.getLogger("urllib3").setLevel(logging.WARNING)
             case 1:
-                logging.getLogger("requests").setLevel(logging.WARNING)
+                logging.getLogger("requests").setLevel(logging.INFO)
-                logging.getLogger("urllib3").setLevel(logging.WARNING)
+                logging.getLogger("urllib3").setLevel(logging.INFO)
                 logger.setLevel(logging.INFO)
             case 2:
                 logger.setLevel(logging.DEBUG)
+                logging.getLogger("requests").setLevel(logging.DEBUG)
+                logging.getLogger("urllib3").setLevel(logging.DEBUG)
             case _:
                 logger.setLevel(logging.INFO)
+                logging.getLogger("requests").setLevel(logging.INFO)
+                logging.getLogger("urllib3").setLevel(logging.INFO)
     return logger
 
 
@@ -218,14 +255,13 @@ def get_api_config(log: Logger, config: str) -> ApiConfig:
     with open(api_config, "rt", encoding="utf-8") as f:
         api_data = yaml.safe_load(f.read())
         servers = [Server(**server) for server in api_data["server"]]
-        login = Login(**(api_data["login"]))
+        api_config_data = ApiConfig(server=servers)
-        api_config_data = ApiConfig(server=servers, login=login)
         log.debug(api_config_data)
         if not api_data:
             log.fatal("API configuration is missing")
             return api_config_data
     for server in api_config_data.server:
-        server.login(api_config_data.login, log)
+        server.login(log)
     with open(api_config, "w", encoding="utf-8") as f:
         yaml.dump(api_data, f)
     return api_config_data

kontor-scripts/sync.py

@@ -25,12 +25,13 @@ parser.add_argument("--cleanup", "-d", action="store_true")
 args = parser.parse_args()
 
 
-def create_item_id_mapping(data_list: List[dict]) -> Dict[str, dict]:
+def create_item_id_mapping(log: Logger, data_list: List[dict]) -> Dict[str, dict]:
     """
     create dictionary with id as key and dictionary as value.
     """
     item_id_mapping: Dict[str, dict] = {}
     for data_item in data_list:
+        log.debug(data_item)        
         item_id_mapping[data_item["id"]] = data_item
     return item_id_mapping
 
@@ -87,7 +88,7 @@ if __name__ == "__main__":
         )
     if len(server_list) > 1:
         for table, path in MAPPING.items():
-            mapping = create_item_id_mapping(export_data[server_list[1].name][table])
+            mapping = create_item_id_mapping(logger, export_data[server_list[1].name][table])
             for item in export_data[server_list[0].name][table]:
                 logger.debug("checking  %s:%s", table, item["id"])
                 check_item_id = item["id"]
@@ -111,8 +112,9 @@ if __name__ == "__main__":
                         )
                 else:
                     logger.info(
-                        "item %s in %s missing", check_item_id, server_list[1].name
+                        "item %s in %s missing: ", check_item_id, server_list[1].name, item
                     )
+                    server_list[1].create(logger, table, item)
             logger.info("synchronization of %s finished", table)
         logger.info("all tables synchronized")
     else: