Raw DCI values saved in history along with corresponding transformed values
authorVictor Kirhenshtein <victor@netxms.org>
Tue, 7 Nov 2017 15:29:02 +0000 (17:29 +0200)
committerVictor Kirhenshtein <victor@netxms.org>
Tue, 7 Nov 2017 15:29:02 +0000 (17:29 +0200)
26 files changed:
ChangeLog
include/netxmsdb.h
include/nms_cscp.h
sql/metadata.in
src/client/java/netxms-client/src/main/java/org/netxms/client/NXCSession.java
src/client/java/netxms-client/src/main/java/org/netxms/client/datacollection/DciDataRow.java
src/java/netxms-eclipse/Dashboard/src/org/netxms/ui/eclipse/dashboard/widgets/ComparisonChartElement.java
src/java/netxms-eclipse/Dashboard/src/org/netxms/ui/eclipse/dashboard/widgets/LineChartElement.java
src/java/netxms-eclipse/PerfView/src/org/netxms/ui/eclipse/perfview/objecttabs/internal/PerfTabGraph.java
src/java/netxms-eclipse/PerfView/src/org/netxms/ui/eclipse/perfview/views/DataComparisonView.java
src/java/netxms-eclipse/PerfView/src/org/netxms/ui/eclipse/perfview/views/HistoricalDataView.java
src/java/netxms-eclipse/PerfView/src/org/netxms/ui/eclipse/perfview/views/HistoricalGraphView.java
src/java/netxms-eclipse/PerfView/src/org/netxms/ui/eclipse/perfview/views/helpers/HistoricalDataLabelProvider.java
src/libnxjava/java/base/netxms-base/src/main/java/org/netxms/base/NXCPCodes.java
src/server/core/dbwrite.cpp
src/server/core/dcitem.cpp
src/server/core/session.cpp
src/server/include/nms_core.h
src/server/tools/nxdbmgr/upgrade_v30.cpp
webui/webapp/Dashboard/src/org/netxms/ui/eclipse/dashboard/widgets/ComparisonChartElement.java
webui/webapp/Dashboard/src/org/netxms/ui/eclipse/dashboard/widgets/LineChartElement.java
webui/webapp/PerfView/src/org/netxms/ui/eclipse/perfview/objecttabs/internal/PerfTabGraph.java
webui/webapp/PerfView/src/org/netxms/ui/eclipse/perfview/views/DataComparisonView.java
webui/webapp/PerfView/src/org/netxms/ui/eclipse/perfview/views/HistoricalDataView.java
webui/webapp/PerfView/src/org/netxms/ui/eclipse/perfview/views/HistoricalGraphView.java
webui/webapp/PerfView/src/org/netxms/ui/eclipse/perfview/views/helpers/HistoricalDataLabelProvider.java

index 9035dcb..35649e8 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,7 @@
 *
 
 - Event groups
+- Raw DCI values saved in history along with corresponding transformed values
 - Fixed issues: NX-900, NX-1102, NX-1199, NX-1219, NX-1289, NX-1292, NX-1332, NX-1345
 
 
index f38bb6c..bb31699 100644 (file)
@@ -25,7 +25,7 @@
 
 #define DB_LEGACY_SCHEMA_VERSION       700
 #define DB_SCHEMA_VERSION_MAJOR        30
-#define DB_SCHEMA_VERSION_MINOR        10
+#define DB_SCHEMA_VERSION_MINOR        11
 
 #define DB_SCHEMA_VERSION_V30_MINOR    DB_SCHEMA_VERSION_MINOR
 
index ea36b0b..5b71f94 100644 (file)
@@ -1214,6 +1214,7 @@ typedef struct
 #define VID_EXPAND_STRING           ((UINT32)605)
 #define VID_ACTION_LIST             ((UINT32)606)
 #define VID_ZONE_SNMP_PORT_COUNT    ((UINT32)607)
+#define VID_INCLUDE_RAW_VALUES      ((UINT32)608)
 
 // Base variabe for single threshold in message
 #define VID_THRESHOLD_BASE          ((UINT32)0x00800000)
index d2263b5..3c2ff8b 100644 (file)
@@ -10,7 +10,7 @@ INSERT INTO metadata (var_name,var_value) VALUES ('SchemaVersionMinor',DB_SCHEMA
 INSERT INTO metadata (var_name,var_value) VALUES ('Syntax',DB_SYNTAX);
 
 INSERT INTO metadata (var_name,var_value)
-       VALUES ('IDataTableCreationCommand','CREATE TABLE idata_%d (item_id integer not null,idata_timestamp integer not null,idata_value varchar(255) null)');
+       VALUES ('IDataTableCreationCommand','CREATE TABLE idata_%d (item_id integer not null,idata_timestamp integer not null,idata_value varchar(255) null,raw_value varchar(255) null)');
 #if defined(DB_MSSQL)
 INSERT INTO metadata (var_name,var_value)
        VALUES ('IDataIndexCreationCommand_0','CREATE CLUSTERED INDEX idx_idata_%d_id_timestamp ON idata_%d(item_id,idata_timestamp)');
index 765ea60..d3ebc02 100644 (file)
@@ -4041,6 +4041,7 @@ public class NXCSession
    {
       final NXCPDataInputStream inputStream = new NXCPDataInputStream(input);
       int rows = 0;
+      DciDataRow row = null;
 
       try
       {
@@ -4052,25 +4053,24 @@ public class NXCSession
 
          for(int i = 0; i < rows; i++)
          {
-            long timestamp = inputStream.readUnsignedInt() * 1000; // convert to
-            // milliseconds
-
+            long timestamp = inputStream.readUnsignedInt() * 1000; // convert to milliseconds
+            Object value;
             switch(dataType)
             {
                case DataCollectionItem.DT_INT:
-                  data.addDataRow(new DciDataRow(new Date(timestamp), new Long(inputStream.readInt())));
+                  value = new Long(inputStream.readInt());
                   break;
                case DataCollectionItem.DT_UINT:
-                  data.addDataRow(new DciDataRow(new Date(timestamp), new Long(inputStream.readUnsignedInt())));
+                  value = new Long(inputStream.readUnsignedInt());
                   break;
                case DataCollectionItem.DT_INT64:
                case DataCollectionItem.DT_UINT64:
                   inputStream.skipBytes(4); // padding
-                  data.addDataRow(new DciDataRow(new Date(timestamp), new Long(inputStream.readLong())));
+                  value = new Long(inputStream.readLong());
                   break;
                case DataCollectionItem.DT_FLOAT:
                   inputStream.skipBytes(4); // padding
-                  data.addDataRow(new DciDataRow(new Date(timestamp), new Double(inputStream.readDouble())));
+                  value = new Double(inputStream.readDouble());
                   break;
                case DataCollectionItem.DT_STRING:
                   StringBuilder sb = new StringBuilder(256);
@@ -4086,9 +4086,23 @@ public class NXCSession
                      sb.append(ch);
                   }
                   inputStream.skipBytes(count * 2);
-                  data.addDataRow(new DciDataRow(new Date(timestamp), sb.toString()));
+                  value = sb.toString();
+                  break;
+               default:
+                  value = null;
                   break;
             }
+            if (timestamp > 0)
+            {
+               row = new DciDataRow(new Date(timestamp), value);
+               data.addDataRow(row);
+            }
+            else
+            {
+               // raw value for previous entry
+               if (row != null)
+                  row.setRawValue(value);
+            }
          }
       }
       catch(IOException e)
@@ -4110,11 +4124,13 @@ public class NXCSession
     * @param from       Start of time range or null for no limit
     * @param to         End of time range or null for no limit
     * @param maxRows    Maximum number of rows to retrieve or 0 for no limit
+    * @param includeRawValues if true raw DCI values will be included into set
     * @return DCI data set
     * @throws IOException  if socket I/O error occurs
     * @throws NXCException if NetXMS server returns an error or operation was timed out
     */
-   private DciData getCollectedDataInternal(long nodeId, long dciId, String instance, String dataColumn, Date from, Date to, int maxRows) throws IOException, NXCException
+   private DciData getCollectedDataInternal(long nodeId, long dciId, String instance, String dataColumn, Date from, Date to,
+         int maxRows, boolean includeRawValues) throws IOException, NXCException
    {
       NXCPMessage msg;
       if (instance != null) // table DCI
@@ -4129,6 +4145,7 @@ public class NXCSession
       }
       msg.setFieldInt32(NXCPCodes.VID_OBJECT_ID, (int) nodeId);
       msg.setFieldInt32(NXCPCodes.VID_DCI_ID, (int) dciId);
+      msg.setField(NXCPCodes.VID_INCLUDE_RAW_VALUES, includeRawValues);
 
       DciData data = new DciData(nodeId, dciId);
 
@@ -4154,7 +4171,8 @@ public class NXCSession
          if (((rowsRemaining == 0) || (rowsRemaining > MAX_DCI_DATA_ROWS)) && (rowsReceived == MAX_DCI_DATA_ROWS))
          {
             // adjust boundaries for next request
-            if (rowsRemaining > 0) rowsRemaining -= rowsReceived;
+            if (rowsRemaining > 0) 
+               rowsRemaining -= rowsReceived;
 
             // Rows goes in newest to oldest order, so if we need to
             // retrieve additional data, we should update timeTo limit
@@ -4184,13 +4202,15 @@ public class NXCSession
     * @param from    Start of time range or null for no limit
     * @param to      End of time range or null for no limit
     * @param maxRows Maximum number of rows to retrieve or 0 for no limit
+    * @param includeRawValues if true raw values will be included into result set
     * @return DCI data set
     * @throws IOException  if socket I/O error occurs
     * @throws NXCException if NetXMS server returns an error or operation was timed out
     */
-   public DciData getCollectedData(long nodeId, long dciId, Date from, Date to, int maxRows) throws IOException, NXCException
+   public DciData getCollectedData(long nodeId, long dciId, Date from, Date to, int maxRows, boolean includeRawValues)
+         throws IOException, NXCException
    {
-      return getCollectedDataInternal(nodeId, dciId, null, null, from, to, maxRows);
+      return getCollectedDataInternal(nodeId, dciId, null, null, from, to, maxRows, includeRawValues);
    }
 
    /**
@@ -4208,13 +4228,12 @@ public class NXCSession
     * @throws IOException  if socket I/O error occurs
     * @throws NXCException if NetXMS server returns an error or operation was timed out
     */
-   public DciData getCollectedTableData(
-      long nodeId, long dciId, String instance, String dataColumn, Date from, Date to, int maxRows)
-      throws IOException, NXCException
+   public DciData getCollectedTableData(long nodeId, long dciId, String instance, String dataColumn, Date from, Date to, int maxRows)
+         throws IOException, NXCException
    {
       if (instance == null || dataColumn == null) 
          throw new NXCException(RCC.INVALID_ARGUMENT);
-      return getCollectedDataInternal(nodeId, dciId, instance, dataColumn, from, to, maxRows);
+      return getCollectedDataInternal(nodeId, dciId, instance, dataColumn, from, to, maxRows, false);
    }
 
    /**
index bd078f8..b2f5dd6 100644 (file)
@@ -29,15 +29,27 @@ public class DciDataRow
 {
        private Date timestamp;
        private Object value;
+       private Object rawValue;
 
        public DciDataRow(Date timestamp, Object value)
        {
                super();
                this.timestamp = timestamp;
                this.value = value;
+               this.rawValue = null;
        }
 
        /**
+        * Set raw value
+        * 
+        * @param rawValue new raw value
+        */
+       public void setRawValue(Object rawValue)
+       {
+          this.rawValue = rawValue;
+       }
+       
+       /**
         * @return the timestamp
         */
        public Date getTimestamp()
@@ -58,7 +70,7 @@ public class DciDataRow
         */
        public String getValueAsString()
        {
-               return value.toString();
+               return (value != null) ? value.toString() : "";
        }
 
        /**
@@ -90,6 +102,50 @@ public class DciDataRow
        }
 
    /**
+    * @return raw value
+    */
+   public Object getRawValue()
+   {
+      return rawValue;
+   }
+   
+   /**
+    * @return raw value as string
+    */
+   public String getRawValueAsString()
+   {
+      return (rawValue != null) ? rawValue.toString() : "";
+   }
+
+   /**
+    * @return raw value as long
+    */
+   public long getRawValueAsLong()
+   {
+      if (rawValue instanceof Long)
+         return ((Long)rawValue).longValue();
+
+      if (rawValue instanceof Double)
+         return ((Double)rawValue).longValue();
+      
+      return 0;
+   }
+
+   /**
+    * @return raw value as double
+    */
+   public double getRawValueAsDouble()
+   {
+      if (rawValue instanceof Long)
+         return ((Long)rawValue).doubleValue();
+
+      if (rawValue instanceof Double)
+         return ((Double)rawValue).doubleValue();
+      
+      return 0;
+   }
+       
+   /**
     * Invert value
     */
    public void invert()
@@ -106,6 +162,6 @@ public class DciDataRow
    @Override
    public String toString()
    {
-      return "DciDataRow [timestamp=" + timestamp + ", value=" + value + "]";
+      return "DciDataRow [timestamp=" + timestamp + ", value=" + value + ", rawValue=" + rawValue + "]";
    }
 }
index b94843a..373035a 100644 (file)
@@ -110,7 +110,7 @@ public abstract class ComparisonChartElement extends ElementWidget
                                for(int i = 0; i < dciList.length; i++)
                                {
                                        if (dciList[i].type == ChartDciConfig.ITEM)
-                                               data[i] = session.getCollectedData(dciList[i].nodeId, dciList[i].dciId, null, null, 1);
+                                               data[i] = session.getCollectedData(dciList[i].nodeId, dciList[i].dciId, null, null, 1, false);
                                        else
                                                data[i] = session.getCollectedTableData(dciList[i].nodeId, dciList[i].dciId, dciList[i].instance, dciList[i].column, null, null, 1);
                                }
index b8836d2..739fc26 100644 (file)
@@ -265,7 +265,7 @@ public class LineChartElement extends ElementWidget
                                {
                                        currentDci = dciList[i];
                                        if (currentDci.type == ChartDciConfig.ITEM)
-                                               data[i] = session.getCollectedData(currentDci.nodeId, currentDci.dciId, from, to, 0);
+                                               data[i] = session.getCollectedData(currentDci.nodeId, currentDci.dciId, from, to, 0, false);
                                        else
                                                data[i] = session.getCollectedTableData(currentDci.nodeId, currentDci.dciId, currentDci.instance, currentDci.column, from, to, 0);
                                }
index 2bca0b7..1b4e206 100644 (file)
@@ -180,7 +180,7 @@ public class PerfTabGraph extends DashboardComposite
                                        for(int i = 0; i < data.length; i++)
                                        {
                                                currentDci = items.get(i);
-                                               data[i] = session.getCollectedData(nodeId, currentDci.getId(), from, to, 0);
+                                               data[i] = session.getCollectedData(nodeId, currentDci.getId(), from, to, 0, false);
                                        }
                                        runInUIThread(new Runnable() {
                                                @Override
index 4506b8e..98e3c9f 100644 (file)
@@ -675,7 +675,7 @@ public class DataComparisonView extends ViewPart
                                {
                                        GraphItem item = items.get(i);
                                        DciData data = (item.getType() == DataCollectionObject.DCO_TYPE_ITEM) ? 
-                                                       session.getCollectedData(item.getNodeId(), item.getDciId(), null, null, 1) :
+                                                       session.getCollectedData(item.getNodeId(), item.getDciId(), null, null, 1, false) :
                                                        session.getCollectedTableData(item.getNodeId(), item.getDciId(), item.getInstance(), item.getDataColumn(), null, null, 1);
                                        DciDataRow value = data.getLastValue();
                                        values[i] = (value != null) ? value.getValueAsDouble() : 0.0;
index 8740cff..220e660 100644 (file)
@@ -73,7 +73,9 @@ public class HistoricalDataView extends ViewPart
        private long dciId;
        private String nodeName;
        private String[] subparts;
-       private String tableName, instance, column;
+       private String tableName;
+       private String instance;
+       private String column;
        private SortableTableViewer viewer;
        private Date timeFrom = null;
        private Date timeTo = null;
@@ -110,10 +112,10 @@ public class HistoricalDataView extends ViewPart
          subparts = parts[1].split("@");
          try
          {
-         dciId = Long.parseLong(subparts[0]);
-         tableName = URLDecoder.decode(subparts[1], "UTF-8"); //$NON-NLS-1$
-         instance = URLDecoder.decode(subparts[2], "UTF-8"); //$NON-NLS-1$
-         column = URLDecoder.decode(subparts[3], "UTF-8"); //$NON-NLS-1$
+            dciId = Long.parseLong(subparts[0]);
+            tableName = URLDecoder.decode(subparts[1], "UTF-8"); //$NON-NLS-1$
+            instance = URLDecoder.decode(subparts[2], "UTF-8"); //$NON-NLS-1$
+            column = URLDecoder.decode(subparts[3], "UTF-8"); //$NON-NLS-1$
          }
          catch(NumberFormatException e)
          {
@@ -125,8 +127,9 @@ public class HistoricalDataView extends ViewPart
          }
       }
       else
+      {
          dciId = Long.parseLong(parts[1]);             
-               
+      }
                setPartName(nodeName + ": [" + (tableName == null ? Long.toString(dciId) : tableName) + "]"); //$NON-NLS-1$ //$NON-NLS-2$
        }
 
@@ -136,8 +139,10 @@ public class HistoricalDataView extends ViewPart
        @Override
        public void createPartControl(Composite parent)
        {
-               final String[] names = { Messages.get().HistoricalDataView_ColTimestamp, Messages.get().HistoricalDataView_ColValue };
-               final int[] widths = { 150, 400 };
+               final String[] names = (tableName != null) ? 
+         new String[] { Messages.get().HistoricalDataView_ColTimestamp, Messages.get().HistoricalDataView_ColValue } :
+         new String[] { Messages.get().HistoricalDataView_ColTimestamp, Messages.get().HistoricalDataView_ColValue, "Raw value" };
+               final int[] widths = { 150, 400, 400 };
                viewer = new SortableTableViewer(parent, names, widths, 0, SWT.DOWN, SWT.FULL_SELECTION | SWT.MULTI);
                viewer.setContentProvider(new ArrayContentProvider());
                viewer.setLabelProvider(new HistoricalDataLabelProvider());
@@ -272,7 +277,7 @@ public class HistoricalDataView extends ViewPart
                           if (subparts != null)
                              data = session.getCollectedTableData(nodeId, dciId, instance, column, timeFrom, timeTo, recordLimit);
                           else
-                             data = session.getCollectedData(nodeId, dciId, timeFrom, timeTo, recordLimit);
+                             data = session.getCollectedData(nodeId, dciId, timeFrom, timeTo, recordLimit, true);
                           
                                runInUIThread(new Runnable() {
                                        @Override
index 3dcf8ec..76a3501 100644 (file)
@@ -441,7 +441,7 @@ public class HistoricalGraphView extends ViewPart implements GraphSettingsChange
                if (currentItem.type == ChartDciConfig.ITEM)
                {
                   data[i] = session.getCollectedData(currentItem.nodeId, currentItem.dciId, settings.getTimeFrom(),
-                        settings.getTimeTo(), 0);
+                        settings.getTimeTo(), 0, false);
                   thresholds[i] = session.getThresholds(currentItem.nodeId, currentItem.dciId);
                }
                else
index 98a5ee6..d269571 100644 (file)
@@ -50,6 +50,8 @@ public class HistoricalDataLabelProvider extends LabelProvider implements ITable
                                return RegionalSettings.getDateTimeFormat().format(((DciDataRow)element).getTimestamp());
                        case 1:
                                return ((DciDataRow)element).getValueAsString();
+         case 2:
+            return ((DciDataRow)element).getRawValueAsString();
                }
                return null;
        }
index 2227c92..5103d87 100644 (file)
@@ -995,6 +995,7 @@ public class NXCPCodes
    public static final long VID_EXPAND_STRING = 605;
    public static final long VID_ACTION_LIST = 606;
    public static final long VID_ZONE_SNMP_PORT_COUNT = 607;
+   public static final long VID_INCLUDE_RAW_VALUES = 608;
 
        public static final long VID_ACL_USER_BASE = 0x00001000L;
        public static final long VID_ACL_USER_LAST = 0x00001FFFL;
index 3fe952a..1fd8bcf 100644 (file)
 #include "nxcore.h"
 
 /**
+ * Delayed SQL request
+ */
+struct DELAYED_SQL_REQUEST
+{
+   TCHAR *query;
+   int bindCount;
+   BYTE *sqlTypes;
+   TCHAR *bindings[1]; /* actual size determined by bindCount field */
+};
+
+/**
+ * Delayed request for idata_ INSERT
+ */
+struct DELAYED_IDATA_INSERT
+{
+   time_t timestamp;
+   UINT32 nodeId;
+   UINT32 dciId;
+   TCHAR rawValue[MAX_RESULT_LENGTH];
+   TCHAR transformedValue[MAX_RESULT_LENGTH];
+};
+
+/**
+ * Delayed request for raw_dci_values UPDATE
+ */
+struct DELAYED_RAW_DATA_UPDATE
+{
+   time_t timestamp;
+   UINT32 dciId;
+   TCHAR rawValue[MAX_RESULT_LENGTH];
+   TCHAR transformedValue[MAX_RESULT_LENGTH];
+};
+
+/**
  * Generic DB writer queue
  */
 Queue *g_dbWriterQueue = NULL;
@@ -117,13 +151,14 @@ void NXCORE_EXPORTABLE QueueSQLRequest(const TCHAR *query, int bindCount, int *s
 /**
  * Queue INSERT request for idata_xxx table
  */
-void QueueIDataInsert(time_t timestamp, UINT32 nodeId, UINT32 dciId, const TCHAR *value)
+void QueueIDataInsert(time_t timestamp, UINT32 nodeId, UINT32 dciId, const TCHAR *rawValue, const TCHAR *transformedValue)
 {
        DELAYED_IDATA_INSERT *rq = (DELAYED_IDATA_INSERT *)malloc(sizeof(DELAYED_IDATA_INSERT));
        rq->timestamp = timestamp;
        rq->nodeId = nodeId;
        rq->dciId = dciId;
-       nx_strncpy(rq->value, value, MAX_RESULT_LENGTH);
+   _tcslcpy(rq->rawValue, rawValue, MAX_RESULT_LENGTH);
+   _tcslcpy(rq->transformedValue, transformedValue, MAX_RESULT_LENGTH);
        g_dciDataWriterQueue->put(rq);
        g_idataWriteRequests++;
 }
@@ -136,8 +171,8 @@ void QueueRawDciDataUpdate(time_t timestamp, UINT32 dciId, const TCHAR *rawValue
        DELAYED_RAW_DATA_UPDATE *rq = (DELAYED_RAW_DATA_UPDATE *)malloc(sizeof(DELAYED_RAW_DATA_UPDATE));
        rq->timestamp = timestamp;
        rq->dciId = dciId;
-       nx_strncpy(rq->rawValue, rawValue, MAX_RESULT_LENGTH);
-       nx_strncpy(rq->transformedValue, transformedValue, MAX_RESULT_LENGTH);
+       _tcslcpy(rq->rawValue, rawValue, MAX_RESULT_LENGTH);
+       _tcslcpy(rq->transformedValue, transformedValue, MAX_RESULT_LENGTH);
        g_dciRawDataWriterQueue->put(rq);
        g_rawDataWriteRequests++;
 }
@@ -207,13 +242,14 @@ static THREAD_RESULT THREAD_CALL IDataWriteThread(void *arg)
                                if (g_dbSyntax == DB_SYNTAX_ORACLE)
                                {
                    TCHAR query[256];
-               _sntprintf(query, 256, _T("INSERT INTO idata_%d (item_id,idata_timestamp,idata_value) VALUES (?,?,?)"), (int)rq->nodeId);
+               _sntprintf(query, 256, _T("INSERT INTO idata_%d (item_id,idata_timestamp,idata_value,raw_value) VALUES (?,?,?,?)"), (int)rq->nodeId);
                DB_STATEMENT hStmt = DBPrepare(hdb, query);
                if (hStmt != NULL)
                {
                   DBBind(hStmt, 1, DB_SQLTYPE_INTEGER, rq->dciId);
                   DBBind(hStmt, 2, DB_SQLTYPE_INTEGER, (INT64)rq->timestamp);
-                  DBBind(hStmt, 3, DB_SQLTYPE_VARCHAR, rq->value, DB_BIND_STATIC);
+                  DBBind(hStmt, 3, DB_SQLTYPE_VARCHAR, rq->transformedValue, DB_BIND_STATIC);
+                  DBBind(hStmt, 4, DB_SQLTYPE_VARCHAR, rq->rawValue, DB_BIND_STATIC);
                   success = DBExecute(hStmt);
                   DBFreeStatement(hStmt);
                }
@@ -225,8 +261,10 @@ static THREAD_RESULT THREAD_CALL IDataWriteThread(void *arg)
                                else
                                {
                TCHAR query[1024];
-               _sntprintf(query, 1024, _T("INSERT INTO idata_%d (item_id,idata_timestamp,idata_value) VALUES (%d,%d,%s)"),
-                          (int)rq->nodeId, (int)rq->dciId, (int)rq->timestamp, (const TCHAR *)DBPrepareString(hdb, rq->value));
+               _sntprintf(query, 1024, _T("INSERT INTO idata_%d (item_id,idata_timestamp,idata_value,raw_value) VALUES (%d,%d,%s,%s)"),
+                          (int)rq->nodeId, (int)rq->dciId, (int)rq->timestamp,
+                          (const TCHAR *)DBPrepareString(hdb, rq->transformedValue),
+                          (const TCHAR *)DBPrepareString(hdb, rq->rawValue));
                success = DBQuery(hdb, query);
                                }
 
index 838fa2f..f4cb8c0 100644 (file)
@@ -709,7 +709,7 @@ bool DCItem::processNewValue(time_t tmTimeStamp, const void *originalValue, bool
 
        // Save transformed value to database
    if ((m_flags & DCF_NO_STORAGE) == 0)
-          QueueIDataInsert(tmTimeStamp, m_owner->getId(), m_id, pValue->getString());
+          QueueIDataInsert(tmTimeStamp, m_owner->getId(), m_id, (const TCHAR *)originalValue, pValue->getString());
    if (g_flags & AF_PERFDATA_STORAGE_DRIVER_LOADED)
       PerfDataStorageRequest(this, tmTimeStamp, pValue->getString());
 
index b9997ec..70ec039 100644 (file)
@@ -3828,7 +3828,7 @@ void ClientSession::sendDCIThresholds(NXCPMessage *request)
 /**
  * Prepare statement for reading data from idata/tdata table
  */
-static DB_STATEMENT PrepareDataSelect(DB_HANDLE hdb, UINT32 nodeId, int dciType, UINT32 maxRows, const TCHAR *condition)
+static DB_STATEMENT PrepareDataSelect(DB_HANDLE hdb, UINT32 nodeId, int dciType, UINT32 maxRows, bool withRawValues, const TCHAR *condition)
 {
        TCHAR query[512];
 
@@ -3836,22 +3836,26 @@ static DB_STATEMENT PrepareDataSelect(DB_HANDLE hdb, UINT32 nodeId, int dciType,
        switch(g_dbSyntax)
        {
                case DB_SYNTAX_MSSQL:
-                       _sntprintf(query, 512, _T("SELECT TOP %d %s_timestamp,%s_value FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC"),
-                                  (int)maxRows, tablePrefix, tablePrefix, tablePrefix, (int)nodeId, condition, tablePrefix);
+                       _sntprintf(query, 512, _T("SELECT TOP %d %s_timestamp,%s_value%s FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC"),
+                                (int)maxRows, tablePrefix, tablePrefix, withRawValues ? _T(",raw_value") : _T(""),
+                                tablePrefix, (int)nodeId, condition, tablePrefix);
                        break;
                case DB_SYNTAX_ORACLE:
-                       _sntprintf(query, 512, _T("SELECT * FROM (SELECT %s_timestamp,%s_value FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC) WHERE ROWNUM<=%d"),
-                                  tablePrefix, tablePrefix, tablePrefix, (int)nodeId, condition, tablePrefix, (int)maxRows);
+                       _sntprintf(query, 512, _T("SELECT * FROM (SELECT %s_timestamp,%s_value%s FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC) WHERE ROWNUM<=%d"),
+                                tablePrefix, tablePrefix, withRawValues ? _T(",raw_value") : _T(""),
+                                tablePrefix, (int)nodeId, condition, tablePrefix, (int)maxRows);
                        break;
                case DB_SYNTAX_MYSQL:
                case DB_SYNTAX_PGSQL:
                case DB_SYNTAX_SQLITE:
-                       _sntprintf(query, 512, _T("SELECT %s_timestamp,%s_value FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC LIMIT %d"),
-                                  tablePrefix, tablePrefix, tablePrefix, (int)nodeId, condition, tablePrefix, (int)maxRows);
+                       _sntprintf(query, 512, _T("SELECT %s_timestamp,%s_value%s FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC LIMIT %d"),
+                                tablePrefix, tablePrefix, withRawValues ? _T(",raw_value") : _T(""),
+                                tablePrefix, (int)nodeId, condition, tablePrefix, (int)maxRows);
                        break;
                case DB_SYNTAX_DB2:
-                  _sntprintf(query, 512, _T("SELECT %s_timestamp,%s_value FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC FETCH FIRST %d ROWS ONLY"),
-                             tablePrefix, tablePrefix, tablePrefix, (int)nodeId, condition, tablePrefix, (int)maxRows);
+                  _sntprintf(query, 512, _T("SELECT %s_timestamp,%s_value%s FROM %s_%d WHERE item_id=?%s ORDER BY %s_timestamp DESC FETCH FIRST %d ROWS ONLY"),
+                           tablePrefix, tablePrefix, withRawValues ? _T(",raw_value") : _T(""),
+                           tablePrefix, (int)nodeId, condition, tablePrefix, (int)maxRows);
                   break;
                default:
                        DbgPrintf(1, _T(">>> INTERNAL ERROR: unsupported database in PrepareDataSelect"));
@@ -3863,7 +3867,7 @@ static DB_STATEMENT PrepareDataSelect(DB_HANDLE hdb, UINT32 nodeId, int dciType,
 /**
  * Get collected data for table or simple DCI
  */
-bool ClientSession::getCollectedDataFromDB(NXCPMessage *request, NXCPMessage *response, DataCollectionTarget *dcTarget, int dciType)
+bool ClientSession::getCollectedDataFromDB(NXCPMessage *request, NXCPMessage *response, DataCollectionTarget *dcTarget, int dciType, bool withRawValues)
 {
    static UINT32 s_rowSize[] = { 8, 8, 16, 16, 516, 16 };
 
@@ -3901,7 +3905,7 @@ bool ClientSession::getCollectedDataFromDB(NXCPMessage *request, NXCPMessage *re
    DCI_DATA_ROW *pCurr;
 
        // If only last value requested, try to get it from cache first
-       if ((maxRows == 1) && (timeTo == 0))
+       if ((maxRows == 1) && (timeTo == 0) && !withRawValues)
        {
           debugPrintf(7, _T("getCollectedDataFromDB: maxRows set to 1, will try to read cached value"));
 
@@ -4037,7 +4041,7 @@ read_from_db:
 
        bool success = false;
        DB_HANDLE hdb = DBConnectionPoolAcquireConnection();
-       DB_STATEMENT hStmt = PrepareDataSelect(hdb, dcTarget->getId(), dciType, maxRows, condition);
+       DB_STATEMENT hStmt = PrepareDataSelect(hdb, dcTarget->getId(), dciType, maxRows, withRawValues, condition);
        if (hStmt != NULL)
        {
                TCHAR dataColumn[MAX_COLUMN_NAME] = _T("");
@@ -4132,6 +4136,48 @@ read_from_db:
                      SwapUCS2String(pCurr->value.string);
                      break;
                }
+
+               if (withRawValues)
+               {
+                  pCurr = (DCI_DATA_ROW *)(((char *)pCurr) + s_rowSize[dataType]);
+                  if (rows == allocated)
+                  {
+                     allocated += 8192;
+                     pData = (DCI_DATA_HEADER *)realloc(pData, allocated * s_rowSize[dataType] + sizeof(DCI_DATA_HEADER));
+                     pCurr = (DCI_DATA_ROW *)(((char *)pData + s_rowSize[dataType] * rows) + sizeof(DCI_DATA_HEADER));
+                  }
+                  rows++;
+
+                  pCurr->timeStamp = 0;   // raw value indicator
+                  switch(dataType)
+                  {
+                     case DCI_DT_INT:
+                     case DCI_DT_UINT:
+                        pCurr->value.int32 = htonl(DBGetFieldULong(hResult, 2));
+                        break;
+                     case DCI_DT_INT64:
+                     case DCI_DT_UINT64:
+                        pCurr->value.ext.v64.int64 = htonq(DBGetFieldUInt64(hResult, 2));
+                        break;
+                     case DCI_DT_FLOAT:
+                        pCurr->value.ext.v64.real = htond(DBGetFieldDouble(hResult, 2));
+                        break;
+                     case DCI_DT_STRING:
+   #ifdef UNICODE
+   #ifdef UNICODE_UCS4
+                        DBGetField(hResult, 2, szBuffer, MAX_DCI_STRING_VALUE);
+                        ucs4_to_ucs2(szBuffer, -1, pCurr->value.string, MAX_DCI_STRING_VALUE);
+   #else
+                        DBGetField(hResult, 2, pCurr->value.string, MAX_DCI_STRING_VALUE);
+   #endif
+   #else
+                        DBGetField(hResult, 2, szBuffer, MAX_DCI_STRING_VALUE);
+                        mb_to_ucs2(szBuffer, -1, pCurr->value.string, MAX_DCI_STRING_VALUE);
+   #endif
+                        SwapUCS2String(pCurr->value.string);
+                        break;
+                  }
+               }
                                }
                                else
                                {
@@ -4228,7 +4274,8 @@ void ClientSession::getCollectedData(NXCPMessage *request)
                        {
                                if (!(g_flags & AF_DB_CONNECTION_LOST))
                                {
-                                       success = getCollectedDataFromDB(request, &msg, (DataCollectionTarget *)object, DCO_TYPE_ITEM);
+                                       success = getCollectedDataFromDB(request, &msg, (DataCollectionTarget *)object,
+                                                DCO_TYPE_ITEM, request->getFieldAsBoolean(VID_INCLUDE_RAW_VALUES));
                                }
                                else
                                {
@@ -4276,7 +4323,7 @@ void ClientSession::getTableCollectedData(NXCPMessage *request)
                        {
                                if (!(g_flags & AF_DB_CONNECTION_LOST))
                                {
-                                       success = getCollectedDataFromDB(request, &msg, (DataCollectionTarget *)object, DCO_TYPE_TABLE);
+                                       success = getCollectedDataFromDB(request, &msg, (DataCollectionTarget *)object, DCO_TYPE_TABLE, false);
                                }
                                else
                                {
index 7a151db..1c7503b 100644 (file)
@@ -551,7 +551,7 @@ private:
    void applyTemplate(NXCPMessage *pRequest);
    void getCollectedData(NXCPMessage *pRequest);
    void getTableCollectedData(NXCPMessage *pRequest);
-       bool getCollectedDataFromDB(NXCPMessage *request, NXCPMessage *response, DataCollectionTarget *object, int dciType);
+       bool getCollectedDataFromDB(NXCPMessage *request, NXCPMessage *response, DataCollectionTarget *object, int dciType, bool withRawValues);
        void clearDCIData(NXCPMessage *pRequest);
        void forceDCIPoll(NXCPMessage *pRequest);
    void changeDCIStatus(NXCPMessage *pRequest);
@@ -823,39 +823,6 @@ public:
 };
 
 /**
- * Delayed SQL request
- */
-typedef struct
-{
-       TCHAR *query;
-       int bindCount;
-       BYTE *sqlTypes;
-       TCHAR *bindings[1]; /* actual size determined by bindCount field */
-} DELAYED_SQL_REQUEST;
-
-/**
- * Delayed request for idata_ INSERT
- */
-typedef struct
-{
-       time_t timestamp;
-       UINT32 nodeId;
-       UINT32 dciId;
-       TCHAR value[MAX_RESULT_LENGTH];
-} DELAYED_IDATA_INSERT;
-
-/**
- * Delayed request for raw_dci_values UPDATE
- */
-typedef struct
-{
-       time_t timestamp;
-       UINT32 dciId;
-       TCHAR rawValue[MAX_RESULT_LENGTH];
-       TCHAR transformedValue[MAX_RESULT_LENGTH];
-} DELAYED_RAW_DATA_UPDATE;
-
-/**
  * Graph ACL entry
  */
 struct GRAPH_ACL_ENTRY
@@ -1038,7 +1005,7 @@ void NXCORE_EXPORTABLE ObjectTransactionEnd();
 
 void NXCORE_EXPORTABLE QueueSQLRequest(const TCHAR *query);
 void NXCORE_EXPORTABLE QueueSQLRequest(const TCHAR *query, int bindCount, int *sqlTypes, const TCHAR **values);
-void QueueIDataInsert(time_t timestamp, UINT32 nodeId, UINT32 dciId, const TCHAR *value);
+void QueueIDataInsert(time_t timestamp, UINT32 nodeId, UINT32 dciId, const TCHAR *rawValue, const TCHAR *transformedValue);
 void QueueRawDciDataUpdate(time_t timestamp, UINT32 dciId, const TCHAR *rawValue, const TCHAR *transformedValue);
 void StartDBWriter();
 void StopDBWriter();
index eee4537..cd2a8d2 100644 (file)
 #include "nxdbmgr.h"
 
 /**
+ * Upgrade from 30.10 to 30.11
+ */
+static bool H_UpgradeFromV10()
+{
+   CHK_EXEC(SQLQuery(_T("UPDATE metadata SET var_value='CREATE TABLE idata_%d (item_id integer not null,idata_timestamp integer not null,idata_value varchar(255) null,raw_value varchar(255) null)' WHERE var_name='IDataTableCreationCommand'")));
+
+   IntegerArray<UINT32> *targets = GetDataCollectionTargets();
+   for(int i = 0; i < targets->size(); i++)
+   {
+      TCHAR query[256];
+      _sntprintf(query, 256, _T("ALTER TABLE idata_%d ADD raw_value varchar(255)"), targets->get(i));
+      CHK_EXEC(SQLQuery(query));
+   }
+   delete targets;
+
+   CHK_EXEC(SetMinorSchemaVersion(11));
+   return true;
+}
+
+/**
  * Upgrade from 30.9 to 30.10
  */
 static bool H_UpgradeFromV9()
@@ -522,6 +542,7 @@ static struct
    bool (* upgradeProc)();
 } s_dbUpgradeMap[] =
 {
+   { 10, 30, 11, H_UpgradeFromV10 },
    { 9, 30, 10, H_UpgradeFromV9 },
    { 8, 30, 9, H_UpgradeFromV8 },
    { 7, 30, 8, H_UpgradeFromV7 },
index b94843a..373035a 100644 (file)
@@ -110,7 +110,7 @@ public abstract class ComparisonChartElement extends ElementWidget
                                for(int i = 0; i < dciList.length; i++)
                                {
                                        if (dciList[i].type == ChartDciConfig.ITEM)
-                                               data[i] = session.getCollectedData(dciList[i].nodeId, dciList[i].dciId, null, null, 1);
+                                               data[i] = session.getCollectedData(dciList[i].nodeId, dciList[i].dciId, null, null, 1, false);
                                        else
                                                data[i] = session.getCollectedTableData(dciList[i].nodeId, dciList[i].dciId, dciList[i].instance, dciList[i].column, null, null, 1);
                                }
index b8836d2..739fc26 100644 (file)
@@ -265,7 +265,7 @@ public class LineChartElement extends ElementWidget
                                {
                                        currentDci = dciList[i];
                                        if (currentDci.type == ChartDciConfig.ITEM)
-                                               data[i] = session.getCollectedData(currentDci.nodeId, currentDci.dciId, from, to, 0);
+                                               data[i] = session.getCollectedData(currentDci.nodeId, currentDci.dciId, from, to, 0, false);
                                        else
                                                data[i] = session.getCollectedTableData(currentDci.nodeId, currentDci.dciId, currentDci.instance, currentDci.column, from, to, 0);
                                }
index 2bca0b7..1b4e206 100644 (file)
@@ -180,7 +180,7 @@ public class PerfTabGraph extends DashboardComposite
                                        for(int i = 0; i < data.length; i++)
                                        {
                                                currentDci = items.get(i);
-                                               data[i] = session.getCollectedData(nodeId, currentDci.getId(), from, to, 0);
+                                               data[i] = session.getCollectedData(nodeId, currentDci.getId(), from, to, 0, false);
                                        }
                                        runInUIThread(new Runnable() {
                                                @Override
index 3855700..ffda273 100644 (file)
@@ -640,7 +640,7 @@ public class DataComparisonView extends ViewPart
                                {
                                        GraphItem item = items.get(i);
                                        DciData data = (item.getType() == DataCollectionObject.DCO_TYPE_ITEM) ? 
-                                                       session.getCollectedData(item.getNodeId(), item.getDciId(), null, null, 1) :
+                                                       session.getCollectedData(item.getNodeId(), item.getDciId(), null, null, 1, false) :
                                                        session.getCollectedTableData(item.getNodeId(), item.getDciId(), item.getInstance(), item.getDataColumn(), null, null, 1);
                                        DciDataRow value = data.getLastValue();
                                        values[i] = (value != null) ? value.getValueAsDouble() : 0.0;
index 8740cff..220e660 100644 (file)
@@ -73,7 +73,9 @@ public class HistoricalDataView extends ViewPart
        private long dciId;
        private String nodeName;
        private String[] subparts;
-       private String tableName, instance, column;
+       private String tableName;
+       private String instance;
+       private String column;
        private SortableTableViewer viewer;
        private Date timeFrom = null;
        private Date timeTo = null;
@@ -110,10 +112,10 @@ public class HistoricalDataView extends ViewPart
          subparts = parts[1].split("@");
          try
          {
-         dciId = Long.parseLong(subparts[0]);
-         tableName = URLDecoder.decode(subparts[1], "UTF-8"); //$NON-NLS-1$
-         instance = URLDecoder.decode(subparts[2], "UTF-8"); //$NON-NLS-1$
-         column = URLDecoder.decode(subparts[3], "UTF-8"); //$NON-NLS-1$
+            dciId = Long.parseLong(subparts[0]);
+            tableName = URLDecoder.decode(subparts[1], "UTF-8"); //$NON-NLS-1$
+            instance = URLDecoder.decode(subparts[2], "UTF-8"); //$NON-NLS-1$
+            column = URLDecoder.decode(subparts[3], "UTF-8"); //$NON-NLS-1$
          }
          catch(NumberFormatException e)
          {
@@ -125,8 +127,9 @@ public class HistoricalDataView extends ViewPart
          }
       }
       else
+      {
          dciId = Long.parseLong(parts[1]);             
-               
+      }
                setPartName(nodeName + ": [" + (tableName == null ? Long.toString(dciId) : tableName) + "]"); //$NON-NLS-1$ //$NON-NLS-2$
        }
 
@@ -136,8 +139,10 @@ public class HistoricalDataView extends ViewPart
        @Override
        public void createPartControl(Composite parent)
        {
-               final String[] names = { Messages.get().HistoricalDataView_ColTimestamp, Messages.get().HistoricalDataView_ColValue };
-               final int[] widths = { 150, 400 };
+               final String[] names = (tableName != null) ? 
+         new String[] { Messages.get().HistoricalDataView_ColTimestamp, Messages.get().HistoricalDataView_ColValue } :
+         new String[] { Messages.get().HistoricalDataView_ColTimestamp, Messages.get().HistoricalDataView_ColValue, "Raw value" };
+               final int[] widths = { 150, 400, 400 };
                viewer = new SortableTableViewer(parent, names, widths, 0, SWT.DOWN, SWT.FULL_SELECTION | SWT.MULTI);
                viewer.setContentProvider(new ArrayContentProvider());
                viewer.setLabelProvider(new HistoricalDataLabelProvider());
@@ -272,7 +277,7 @@ public class HistoricalDataView extends ViewPart
                           if (subparts != null)
                              data = session.getCollectedTableData(nodeId, dciId, instance, column, timeFrom, timeTo, recordLimit);
                           else
-                             data = session.getCollectedData(nodeId, dciId, timeFrom, timeTo, recordLimit);
+                             data = session.getCollectedData(nodeId, dciId, timeFrom, timeTo, recordLimit, true);
                           
                                runInUIThread(new Runnable() {
                                        @Override
index dcd0a10..426d040 100644 (file)
@@ -433,7 +433,7 @@ public class HistoricalGraphView extends ViewPart implements GraphSettingsChange
                if (currentItem.type == ChartDciConfig.ITEM)
                {
                   data[i] = session.getCollectedData(currentItem.nodeId, currentItem.dciId, settings.getTimeFrom(),
-                        settings.getTimeTo(), 0);
+                        settings.getTimeTo(), 0, false);
                   thresholds[i] = session.getThresholds(currentItem.nodeId, currentItem.dciId);
                }
                else
index 98a5ee6..d269571 100644 (file)
@@ -50,6 +50,8 @@ public class HistoricalDataLabelProvider extends LabelProvider implements ITable
                                return RegionalSettings.getDateTimeFormat().format(((DciDataRow)element).getTimestamp());
                        case 1:
                                return ((DciDataRow)element).getValueAsString();
+         case 2:
+            return ((DciDataRow)element).getRawValueAsString();
                }
                return null;
        }