Commit c12d75a0 authored by Christian Reuschling's avatar Christian Reuschling
Browse files

clustering and topic timeline capabilities

parent 24e3f78e
......@@ -5,12 +5,11 @@
<groupId>dfki.sds.dynaq</groupId>
<artifactId>dynaq</artifactId>
<packaging>jar</packaging>
<version>2.5-SNAPSHOT</version>
<version>2.6-SNAPSHOT</version>
<name>dynaq</name>
<url>http://dynaq.opendfki.de</url>
<developers>
<developer>
<name>Christian Reuschling</name>
......@@ -21,14 +20,14 @@
</developers>
<properties>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<!-- <j2seVersion>1.7</j2seVersion> -->
<maven.compiler.encoding>UTF-8</maven.compiler.encoding>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<additionalparam>-Xdoclint:none</additionalparam>
<maven.build.timestamp.format>yyyy.MM.dd HH:mm</maven.build.timestamp.format>
......@@ -72,7 +71,6 @@
</dependencyManagement>
<dependencies>
<dependency>
......@@ -82,25 +80,25 @@
<scope>test</scope>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.junit.jupiter</groupId>-->
<!-- <artifactId>junit-jupiter</artifactId>-->
<!-- <version>5.7.1</version>-->
<!-- <scope>test</scope>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.junit.vintage</groupId>-->
<!-- <artifactId>junit-vintage-engine</artifactId>-->
<!-- <version>5.7.1</version>-->
<!-- <scope>test</scope>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.junit.jupiter</groupId>-->
<!-- <artifactId>junit-jupiter</artifactId>-->
<!-- <version>5.7.1</version>-->
<!-- <scope>test</scope>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.junit.vintage</groupId>-->
<!-- <artifactId>junit-vintage-engine</artifactId>-->
<!-- <version>5.7.1</version>-->
<!-- <scope>test</scope>-->
<!-- </dependency>-->
<!-- leechcrawler-->
<dependency>
<groupId>de.dfki.sds</groupId>
<artifactId>leechcrawler</artifactId>
<version>1.25.1</version>
<version>1.26.1</version>
<exclusions>
<exclusion>
<artifactId>asm-debug-all</artifactId>
......@@ -113,33 +111,33 @@
<artifactId>commons-io</artifactId>
<version>2.8.0</version>
</dependency>
<!-- leechcrawler-->
<dependency>
<groupId>de.dfki.sds</groupId>
<artifactId>leechprojects</artifactId>
<version>0.3-SNAPSHOT</version>
</dependency>
<!-- leechcrawler-->
<dependency>
<groupId>com.jgoodies</groupId>
<artifactId>jgoodies-binding</artifactId>
<version>2.6.1</version>
<!-- <version>2.13.0</version>-->
<!-- <version>2.13.0</version>-->
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.google.code.gson</groupId>-->
<!-- <artifactId>gson</artifactId>-->
<!-- <version>2.8.6</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.esotericsoftware</groupId>-->
<!-- <artifactId>kryo</artifactId>-->
<!-- <version>4.0.0</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.google.code.gson</groupId>-->
<!-- <artifactId>gson</artifactId>-->
<!-- <version>2.8.6</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.esotericsoftware</groupId>-->
<!-- <artifactId>kryo</artifactId>-->
<!-- <version>4.0.0</version>-->
<!-- </dependency>-->
<dependency>
......@@ -156,6 +154,7 @@
</dependency>
<!-- Lucene -->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
......@@ -190,43 +189,30 @@
<version>4.9.0</version>
</dependency>
<!-- Lucene-->
<!-- <dependency> -->
<!-- <groupId>org.apache.lucene</groupId> -->
<!-- <artifactId>lucene-spatial</artifactId> -->
<!-- <version>4.9.0</version> -->
<!-- </dependency> -->
<!-- <dependency>-->
<!-- <groupId>de.dfki.km</groupId>-->
<!-- <artifactId>inquisition</artifactId>-->
<!-- <version>23-SNAPSHOT</version>-->
<!-- </dependency>-->
<dependency>
<groupId>de.dfki.sds</groupId>
<artifactId>kafka</artifactId>
<version>1.3</version>
<groupId>org.carrot2</groupId>
<artifactId>carrot2-core</artifactId>
<version>4.2.1</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>de.dfki.km</groupId>-->
<!-- <artifactId>thumbnailer</artifactId>-->
<!-- <version>23-SNAPSHOT</version>-->
<!-- </dependency>-->
<!-- <dependency> -->
<!-- <groupId>dfki.km.delight</groupId> -->
<!-- <artifactId>xmlrpc-delight</artifactId> -->
<!-- <version>2.6.3</version> -->
<!-- </dependency> -->
<dependency>
<!-- Wir nehmen eine geshadede Version um Versionskonflikte mit Lucene zu vermeiden-->
<!-- <groupId>org.carrot2.lang</groupId>-->
<!-- <artifactId>carrot2-lang-lucene</artifactId>-->
<groupId>org.dynaq.shaded.org.carrot2.lang</groupId>
<artifactId>carrot2-lang-lucene-shaded</artifactId>
<version>4.2.1</version>
</dependency>
<!-- <dependency> -->
<!-- <groupId>dfki.km.delight</groupId> -->
<!-- <artifactId>xstream-converter</artifactId> -->
<!-- <version>1.0.1</version> -->
<!-- </dependency> -->
<dependency>
<groupId>de.dfki.sds</groupId>
<artifactId>kafka</artifactId>
<version>1.4</version>
</dependency>
<dependency>
......@@ -244,8 +230,6 @@
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId>
......@@ -259,14 +243,6 @@
</dependency>
<dependency>
<groupId>javax.jmdns</groupId>
<artifactId>jmdns</artifactId>
......@@ -293,8 +269,6 @@
</dependency>
<dependency>
<groupId>org.piccolo2d</groupId>
<artifactId>piccolo2d-core</artifactId>
......@@ -314,28 +288,39 @@
</dependency>
<dependency>
<groupId>skinlf</groupId>
<artifactId>skinlf</artifactId>
<version>1.2.3</version>
</dependency>
<!-- braucht man für den document graph-->
<dependency>
<groupId>org.prefuse</groupId>
<artifactId>prefuse</artifactId>
<version>beta-20071021</version>
</dependency>
<!-- cluster visualization-->
<dependency>
<groupId>org.graphstream</groupId>
<artifactId>gs-core</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>org.graphstream</groupId>
<artifactId>gs-ui-swing</artifactId>
<version>2.0</version>
</dependency>
</dependencies>
<dependency>
<groupId>org.jfree</groupId>
<artifactId>jfreechart</artifactId>
<version>1.5.3</version>
</dependency>
</dependencies>
<build>
......@@ -349,7 +334,8 @@
<version>2.1.0</version>
<configuration>
<assembleDirectory>${project.build.directory}/assembleDir</assembleDirectory>
<extraJvmArguments>-Xmx2G --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.desktop/sun.awt.dnd=ALL-UNNAMED</extraJvmArguments>
<extraJvmArguments>-Xmx2G --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.desktop/sun.awt.dnd=ALL-UNNAMED --illegal-access=permit -Dsun.java2d.uiScale=100% -Dkafka.baseDirectory=./kafkaBaseDir
</extraJvmArguments>
<binFolder>/bin</binFolder>
<repositoryName>lib</repositoryName>
<repositoryLayout>flat</repositoryLayout>
......@@ -365,14 +351,14 @@
<mainClass>org.dynaq.DynaQ</mainClass>
<name>dynaq</name>
</program>
<program>
<mainClass>org.dynaq.webservice.DynaQServerManager</mainClass>
<name>runServer</name>
</program>
<program>
<mainClass>org.dynaq.project_specific.aloe.DynaQServerManager4Aloe</mainClass>
<name>runAloeObserver</name>
</program>
<!-- <program>-->
<!-- <mainClass>org.dynaq.webservice.DynaQServerManager</mainClass>-->
<!-- <name>runServer</name>-->
<!-- </program>-->
<!-- <program>-->
<!-- <mainClass>org.dynaq.project_specific.aloe.DynaQServerManager4Aloe</mainClass>-->
<!-- <name>runAloeObserver</name>-->
<!-- </program>-->
<program>
<mainClass>org.dynaq.index.Indexer</mainClass>
<name>createIndex</name>
......@@ -390,60 +376,58 @@
</plugin>
<!-- <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>webstart-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>jnlp-inline</goal>
</goals>
</execution>
</executions>
<configuration>
<libPath>lib</libPath>
<pack200>false</pack200>
<gzip>false</gzip>
<outputJarVersions>false</outputJarVersions>
<unsignAlreadySignedJars>true</unsignAlreadySignedJars>
<verbose>false</verbose>
<j2seVersion>1.6</j2seVersion>
<jnlp>
<inputTemplateResourcePath>${project.basedir}</inputTemplateResourcePath>
<inputTemplate>src/main/jnlp/template.vm</inputTemplate>
<outputFile>launch.jnlp</outputFile>
<mainClass>org.dynaq.DynaQ</mainClass>
</jnlp>
<sign>
<keystore>keystore</keystore>
<keypass>keypass</keypass>
<storepass>storepass</storepass>
<alias>dynaq_sign</alias>
<validity>3650</validity>
<dnameCn>dynaq.opendfki.de</dnameCn>
<dnameOu>SDS/dnameOu>
<dnameO>DFKI GmbH</dnameO>
<dnameL>Rhineland Palatinate</dnameL>
<dnameSt>Kaiserslautern</dnameSt>
<dnameC>Germany</dnameC>
<verify>true</verify>
<keystoreConfig>
<delete>false</delete>
<gen>true</gen>
</keystoreConfig>
</sign>
</configuration>
</plugin> -->
<!-- <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>webstart-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>jnlp-inline</goal>
</goals>
</execution>
</executions>
<configuration>
<libPath>lib</libPath>
<pack200>false</pack200>
<gzip>false</gzip>
<outputJarVersions>false</outputJarVersions>
<unsignAlreadySignedJars>true</unsignAlreadySignedJars>
<verbose>false</verbose>
<j2seVersion>1.6</j2seVersion>
<jnlp>
<inputTemplateResourcePath>${project.basedir}</inputTemplateResourcePath>
<inputTemplate>src/main/jnlp/template.vm</inputTemplate>
<outputFile>launch.jnlp</outputFile>
<mainClass>org.dynaq.DynaQ</mainClass>
</jnlp>
<sign>
<keystore>keystore</keystore>
<keypass>keypass</keypass>
<storepass>storepass</storepass>
<alias>dynaq_sign</alias>
<validity>3650</validity>
<dnameCn>dynaq.opendfki.de</dnameCn>
<dnameOu>SDS/dnameOu>
<dnameO>DFKI GmbH</dnameO>
<dnameL>Rhineland Palatinate</dnameL>
<dnameSt>Kaiserslautern</dnameSt>
<dnameC>Germany</dnameC>
<verify>true</verify>
<keystoreConfig>
<delete>false</delete>
<gen>true</gen>
</keystoreConfig>
</sign>
</configuration>
</plugin> -->
<plugin>
......@@ -458,8 +442,8 @@
</archive>
</configuration>
</plugin>
<plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.2.1</version>
......@@ -477,8 +461,6 @@
</plugins>
<pluginManagement>
<plugins>
<plugin>
......@@ -489,9 +471,9 @@
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
<encoding>${maven.compiler.encoding}</encoding>
<compilerArgs>
<compilerArgs>
<arg>-parameters</arg>
</compilerArgs>
</compilerArgs>
</configuration>
</plugin>
......@@ -509,8 +491,8 @@
</build>
<profiles>
<profile>
<id>disable-java8-doclint</id>
<activation>
......@@ -521,11 +503,7 @@
</properties>
</profile>
</profiles>
</project>
......@@ -3,6 +3,15 @@ package org.dynaq;
import de.dfki.inquisitor.exceptions.ExceptionUtils;
import de.dfki.inquisitor.file.FileUtilz;
import de.dfki.inquisitor.resources.ResourceUtilz;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.NativeFSLockFactory;
import org.kafkaRCP.core.KafkaRCPConstants;
import org.kafkaRCP.ui.KafkaRCP;
import javax.swing.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
......@@ -10,19 +19,6 @@ import java.util.Locale;
import java.util.logging.LogManager;
import java.util.logging.Logger;
import javax.swing.JOptionPane;
import javax.swing.ToolTipManager;
import javax.swing.UIManager;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.NativeFSLockFactory;
import org.kafkaRCP.core.KafkaRCPConstants;
import org.kafkaRCP.ui.KafkaRCP;
import de.dfki.inquisitor.exceptions.ExceptionUtils;
import de.dfki.inquisitor.file.FileUtilz;
import de.dfki.inquisitor.resources.ResourceUtilz;
......
......@@ -3,26 +3,13 @@ package org.dynaq.config;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
import de.dfki.inquisitor.collections.ConfigurationException;
import de.dfki.inquisitor.collections.ConfigurationValue;
import de.dfki.inquisitor.collections.MultiValueConfiguration;
import de.dfki.inquisitor.collections.MultiValueLinkedHashMap;
import de.dfki.inquisitor.file.FileUtilz;
import de.dfki.inquisitor.text.DateParser;
import de.dfki.km.leech.metadata.LeechMetadata;
import org.apache.lucene.document.FieldType.NumericType;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
......@@ -40,21 +27,18 @@ import org.dynaq.core.DynaQException;
import org.dynaq.index.LuceneIndexSet;
import org.dynaq.util.DateUtils;
import org.dynaq.util.lucene.basic.Buzzwords;
// import org.dynaq.util.lucene.basic.DynamicFieldType;
import org.dynaq.util.lucene.basic.DynaQDefaultFieldConfig;
import org.dynaq.util.lucene.basic.DynamicFieldType;
import org.dynaq.util.lucene.basic.FieldConfig;
import org.kafkaRCP.core.KafkaRCPConstants;
import de.dfki.inquisitor.collections.ConfigurationException;
import de.dfki.inquisitor.collections.ConfigurationValue;
import de.dfki.inquisitor.collections.MultiValueConfiguration;
import de.dfki.inquisitor.collections.MultiValueLinkedHashMap;
import de.dfki.inquisitor.file.FileUtilz;
import de.dfki.inquisitor.text.DateParser;
import de.dfki.km.leech.lucene.LeechDefaultFieldConfig;
import de.dfki.km.leech.metadata.LeechMetadata;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
......@@ -203,21 +187,21 @@ public class AttributeConfig
*
* @author Christian Reuschling, Dipl.Ing.(BA)
*/
public static interface AttributeTypes
public interface AttributeTypes
{
static public final String String = "STRING";
String String = "STRING";
static public final String Date = "DATE";
String Date = "DATE";
static public final String Time = "TIME";
String Time = "TIME";
static public final String Integer = "INTEGER";
String Integer = "INTEGER";
static public final String Long = "LONG";
String Long = "LONG";
static public final String Float = "FLOAT";
String Float = "FLOAT";
static public final String Double = "DOUBLE";
String Double = "DOUBLE";
}
......@@ -228,29 +212,29 @@ public class AttributeConfig
*
* @author Christian Reuschling, Dipl.Ing.(BA)
*/
public static interface ConfigAttributes
public interface ConfigAttributes
{
/**
* dynaqService.conf
*/
static public interface DYNAQSERVICE
interface DYNAQSERVICE
{
static public final String DEFAULT_INDEX_URIORPATH = "defaultIndexUriOrPath";
String DEFAULT_INDEX_URIORPATH = "defaultIndexUriOrPath";
static public final String ENABLE_REMOTE_ACCESS = "enableRemoteAccess";
String ENABLE_REMOTE_ACCESS = "enableRemoteAccess";
static public final String EXTERNAL_INDEX_URIORPATH = "externalIndexUriOrPath";
String EXTERNAL_INDEX_URIORPATH = "externalIndexUriOrPath";
static public final String INDEXSET = "indexSet";
String INDEXSET = "indexSet";
static public final String SERVICE_ODBMS_PATH = "serviceODBMSPath";
String SERVICE_ODBMS_PATH = "serviceODBMSPath";
static public final String SERVICE_PORT = "port";
String SERVICE_PORT = "port";
static public final String SERVICE_ONLY_LOCALHOST_VISIBLE = "onlyLocalhostVisible";
String SERVICE_ONLY_LOCALHOST_VISIBLE = "onlyLocalhostVisible";
static public final String SERVICE_RDBMS_PATH = "serviceRDBMSPath";
String SERVICE_RDBMS_PATH = "serviceRDBMSPath";
}
......@@ -259,24 +243,24 @@ public class AttributeConfig
/**
* indexerSources.conf
*/
static public interface IMAP_DIRECTORY_ATTS
interface IMAP_DIRECTORY_ATTS
{
static public final String DIRECTORY = "directory";
String DIRECTORY = "directory";
static public final String PASSWORD =