Add image suggestion feature to iMatrics widget

scripts/extensions/auto-tagging-widget/src/ImageTaggingComponent/ImageTaggingComponent.tsx

@@ -0,0 +1,278 @@
+import * as React from "react";
+import { ISuperdesk } from "superdesk-api";
+import { ToggleBoxNext } from "superdesk-ui-framework";
+
+interface ITag {
+  title: string;
+  type: string;
+  pubStatus: boolean;
+  weight: number;
+}
+
+interface IImage {
+  imageUrl: string;
+  thumbnailUrl: string;
+  id?: string;
+  headline?: string;
+  caption?: string;
+  credit?: string;
+  byline?: string;
+  source?: string;
+  dateCreated?: string;
+  archivedTime?: string;
+}
+
+interface IProps {
+  superdesk: ISuperdesk;
+  data: any;
+  style?: React.CSSProperties;
+}
+
+const gridContainerStyle: React.CSSProperties = {
+  display: "grid",
+  gridTemplateColumns: "3fr 1fr",
+  gridColumnGap: "4px",
+  maxHeight: "320px",
+  overflow: "hidden",
+};
+
+const gridItemLeftStyle: React.CSSProperties = {
+  width: "100%",
+  height: "100%",
+  maxHeight: "inherit",
+};
+
+const gridItemRightStyle: React.CSSProperties = {
+  width: "100%",
+  height: "100%",
+  maxHeight: "inherit",
+};
+
+const imageContentStyle: React.CSSProperties = {
+  width: "100%",
+  maxHeight: "inherit",
+  overflowY: "auto",
+  display: "grid",
+  gridRowGap: "4px",
+};
+
+const imageStyle: React.CSSProperties = {
+  width: "100%",
+  height: "auto",
+};
+
+const imageWrapperStyle: React.CSSProperties = {
+  cursor: "pointer",
+  maxHeight: "100%",
+};
+
+const selectedCardStyle: React.CSSProperties = {
+  display: "flex",
+  flexDirection: "column",
+  height: "100%",
+  overflow: "hidden",
+  boxShadow: "1px 3px 6px 0 rgba(0,0,0,0.2)",
+  width: "100%",
+  borderRadius: "4px",
+};
+
+const cardStyle: React.CSSProperties = {
+  boxShadow: "1px 3px 6px 0 rgba(0,0,0,0.2)",
+  width: "100%",
+  borderRadius: "2px",
+};
+
+let isMounted: boolean = false;
+
+const ImageTaggingComponent = (props: IProps) => {
+  const { superdesk, data, style } = props;
+  const { httpRequestJsonLocal } = superdesk;
+
+  const [showImages] = React.useState<boolean>(true);
+  const [isLoading, setIsLoading] = React.useState<boolean>(false);
+  const [fetchError, setFetchError] = React.useState<boolean>(false);
+  const [selectedImage, setSelectedImage] = React.useState<IImage | null>(null);
+  const [images, setImages] = React.useState<Array<IImage>>([]);
+
+  const fetchImages = async (item: Array<ITag>, signal: AbortSignal) => {
+    setIsLoading(true);
+    httpRequestJsonLocal({
+      abortSignal: signal,
+      method: "POST",
+      path: "/ai_image_suggestions/",
+      payload: {
+        service: "imatrics",
+        item,
+      },
+    })
+      .then((res: any) => {
+        if (isMounted) {
+          setFetchError(false);
+          try {
+            setSelectedImage(res.result[0]);
+            setImages(res.result ?? []);
+          } catch {
+            setSelectedImage(null);
+            setImages([]);
+          }
+        }
+      })
+      .catch(() => {
+        setFetchError(true);
+      })
+      .finally(() => isMounted && setIsLoading(false));
+  };
+
+  const formatTags = (concepts: any) => {
+    let res: Array<ITag> = [];
+    for (const key in concepts) {
+      if (key === "subject") {
+        concepts[key].forEach((concept: any) => {
+          res.push({
+            title: concept.name,
+            type: "category",
+            pubStatus: true,
+            weight: 1,
+          });
+        });
+      } else {
+        concepts[key].forEach((concept: any) => {
+          res.push({
+            title: concept.name,
+            type: key,
+            pubStatus: true,
+            weight: 1,
+          });
+        });
+      }
+    }
+    return res;
+  };
+
+  const handleClickImage = (image: IImage) => {
+    setSelectedImage(image);
+  };
+
+  const handleDragStart = (
+    event: React.DragEvent<HTMLDivElement>,
+    image: any,
+  ) => {
+    try {
+      event.dataTransfer.setData(
+        "application/superdesk.item.picture",
+        JSON.stringify({
+          _id: image.id ?? "",
+          guid: image.id ?? "",
+          description_text: image.caption ?? "",
+          headline: image.headline ?? "",
+          original_source: image.source ?? "",
+          source: image.source ?? "",
+          versioncreated: image.archivedTime ?? "",
+          firstcreated: image.dateCreated ?? "",
+          pubstatus: "usable",
+          _type: "externalsource",
+          fetch_endpoint: "scanpix",
+          mimetype: "image/jpeg",
+          type: "picture",
+          renditions: {
+            thumbnail: {
+              href: image.thumbnailUrl ?? "",
+            },
+            viewImage: {
+              href: image.imageUrl ?? "",
+            },
+            baseImage: {
+              href: image.imageUrl ?? "",
+            },
+          },
+          byline: image.byline ?? "",
+          _created: image.dateCreated ?? "",
+        }),
+      );
+    } catch {
+      return;
+    }
+  };
+
+  React.useEffect(() => {
+    const currentTags = data;
+    const formattedTags = formatTags(currentTags);
+    const controller = new AbortController();
+    fetchImages(formattedTags, controller.signal);
+    return () => controller.abort();
+  }, [data]);
+
+  React.useEffect(() => {
+    isMounted = true;
+    return () => {
+      isMounted = false;
+    };
+  }, []);
+
+  return isMounted ? (
+    <ToggleBoxNext
+      title={`image suggestions ${
+        isLoading ? "(...)" : fetchError ? "(error)" : `(${images.length})`
+      }`}
+      style="circle"
+      isOpen={showImages}
+      key="image-suggestion"
+    >
+      <div style={style}>
+        {isLoading ? (
+          <div style={{ display: "flex", alignItems: "center" }}>
+            <div className="spinner-big" />
+          </div>
+        ) : images?.length > 0 ? (
+          <div style={gridContainerStyle}>
+            <div style={gridItemLeftStyle}>
+              <figure style={selectedCardStyle}>
+                <div style={{ maxHeight: "72%" }}>
+                  <img
+                    style={imageStyle}
+                    alt=""
+                    src={selectedImage?.imageUrl}
+                    onDragStart={(e) => handleDragStart(e, selectedImage)}
+                  />
+                </div>
+                <figcaption
+                  style={{
+                    padding: "4px",
+                    overflow: "auto",
+                    color: "#000",
+                    backgroundColor: "rgba(255,255,255,0.75)",
+                    height: "100%",
+                  }}
+                >
+                  {selectedImage?.caption}
+                </figcaption>
+              </figure>
+            </div>
+            <div style={gridItemRightStyle}>
+              <div style={imageContentStyle}>
+                {images.map((image: IImage, i: number) => (
+                  <div key={i} style={cardStyle}>
+                    <div style={imageWrapperStyle}>
+                      <img
+                        style={imageStyle}
+                        key={i}
+                        alt=""
+                        src={image.imageUrl}
+                        onClick={() => {
+                          handleClickImage(image);
+                        }}
+                        onDragStart={(e) => handleDragStart(e, image)}
+                      />
+                    </div>
+                  </div>
+                ))}
+              </div>
+            </div>
+          </div>
+        ) : null}
+      </div>
+    </ToggleBoxNext>
+  ) : null;
+};
+
+export default ImageTaggingComponent;

scripts/extensions/auto-tagging-widget/src/auto-tagging.tsx

@@ -15,6 +15,8 @@ import {getAutoTaggingVocabularyLabels} from './common';
 import {getExistingTags, createTagsPatch} from './data-transformations';
 import {noop} from 'lodash';
 
+import ImageTaggingComponent from './ImageTaggingComponent/ImageTaggingComponent';
+
 export const entityGroups = OrderedSet(['place', 'person', 'organisation']);
 
 export type INewItem = Partial<ITagUi>;
@@ -47,13 +49,17 @@ interface IIMatricsFields {
     };
 }
 
-type IEditableData = {original: IAutoTaggingResponse; changes: IAutoTaggingResponse};
+type IEditableData = {
+    original: IAutoTaggingResponse; 
+    changes: IAutoTaggingResponse;
+};
 
 interface IState {
     runAutomaticallyPreference: boolean | 'loading';
     data: 'not-initialized' | 'loading' | IEditableData;
     newItem: INewItem | null;
     vocabularyLabels: Map<string, string> | null;
+    showImageWidget: boolean;
 }
 
 const RUN_AUTOMATICALLY_PREFERENCE = 'run_automatically';
@@ -163,7 +169,6 @@ export function getAutoTaggingComponent(superdesk: ISuperdesk, label: string) {
         private isDirty: (a: IAutoTaggingResponse, b: Partial<IAutoTaggingResponse>) => boolean;
         private _mounted: boolean;
         private iMatricsFields = superdesk.instance.config.iMatricsFields ?? {entities: {}, others: {}};
-
         constructor(props: IProps) {
             super(props);
 
@@ -172,6 +177,7 @@ export function getAutoTaggingComponent(superdesk: ISuperdesk, label: string) {
                 newItem: null,
                 runAutomaticallyPreference: 'loading',
                 vocabularyLabels: null,
+                showImageWidget: true,
             };
 
             this._mounted = false;
@@ -351,7 +357,6 @@ export function getAutoTaggingComponent(superdesk: ISuperdesk, label: string) {
         }
         componentDidMount() {
             this._mounted = true;
-
             Promise.all([
                 getAutoTaggingVocabularyLabels(superdesk),
                 preferences.get(RUN_AUTOMATICALLY_PREFERENCE),
@@ -368,7 +373,7 @@ export function getAutoTaggingComponent(superdesk: ISuperdesk, label: string) {
             this._mounted = false;
         }
         render() {
-            const {runAutomaticallyPreference, vocabularyLabels} = this.state;
+            const {runAutomaticallyPreference, vocabularyLabels, showImageWidget} = this.state;
 
             if (runAutomaticallyPreference === 'loading' || vocabularyLabels == null) {
                 return null;
@@ -454,7 +459,7 @@ export function getAutoTaggingComponent(superdesk: ISuperdesk, label: string) {
                     <div className="widget-content sd-padding-all--2">
                         <div>
                             <div className="form__row form__row--flex sd-padding-b--1">
-                                <ButtonGroup align="start">
+                                <ButtonGroup align="left">
                                     <Switch
                                         value={runAutomaticallyPreference}
                                         disabled={readOnly}
@@ -706,6 +711,10 @@ export function getAutoTaggingComponent(superdesk: ISuperdesk, label: string) {
 
                                         <div className="widget-content__main">
                                             {allGroupedAndSorted.map((item) => item).toArray()}
+                                            {this.state.showImageWidget && <ImageTaggingComponent
+                                                superdesk={superdesk}
+                                                data={toServerFormat(data.changes.analysis, superdesk)}
+                                            />}
                                         </div>
                                     </React.Fragment>
                                 );