Collected DCI data recalculation based on stored raw values and current transformatio...
authorVictor Kirhenshtein <victor@netxms.org>
Wed, 8 Nov 2017 16:16:45 +0000 (18:16 +0200)
committerVictor Kirhenshtein <victor@netxms.org>
Wed, 8 Nov 2017 16:16:45 +0000 (18:16 +0200)
21 files changed:
ChangeLog
include/nms_cscp.h
src/client/java/netxms-client/src/main/java/org/netxms/client/NXCSession.java
src/java/netxms-eclipse/DataCollection/plugin.xml
src/java/netxms-eclipse/DataCollection/src/org/netxms/ui/eclipse/datacollection/actions/StartDataRecalculation.java [new file with mode: 0644]
src/libnetxms/nxcp.cpp
src/libnxjava/java/base/netxms-base/src/main/java/org/netxms/base/NXCPCodes.java
src/server/core/Makefile.am
src/server/core/Makefile.w32
src/server/core/datacoll.cpp
src/server/core/dci_recalc.cpp [new file with mode: 0644]
src/server/core/dcitem.cpp
src/server/core/dcobject.cpp
src/server/core/dctarget.cpp
src/server/core/session.cpp
src/server/include/nms_core.h
src/server/include/nms_dcoll.h
src/server/include/nms_objects.h
src/server/include/nxcore_jobs.h
webui/webapp/DataCollection/plugin.xml
webui/webapp/DataCollection/src/org/netxms/ui/eclipse/datacollection/actions/StartDataRecalculation.java [new file with mode: 0644]

index 5073c36..f9f659d 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,7 @@
 
 - Event groups
 - Raw DCI values saved in history along with corresponding transformed values
+- Collected DCI data recalculation based on stored raw values and current transformation settings
 - Fixed issues: NX-900, NX-1102, NX-1199, NX-1219, NX-1289, NX-1292, NX-1332, NX-1345
 
 
index 67c3259..a06d9e5 100644 (file)
@@ -247,7 +247,7 @@ typedef struct
 #define CMD_MODIFY_OBJECT                 0x0008
 #define CMD_OBJECT_LIST_END               0x0009
 #define CMD_OBJECT_UPDATE                 0x000A
-//unused: #define CMD_GET_EVENTS                    0x000B
+#define CMD_RECALCULATE_DCI_VALUES        0x000B
 #define CMD_EVENTLOG_RECORDS              0x000C
 #define CMD_GET_CONFIG_VARLIST            0x000D
 #define CMD_SET_CONFIG_VARIABLE           0x000E
index e7f6718..f720528 100644 (file)
@@ -4260,6 +4260,25 @@ public class NXCSession
    }
    
    /**
+    * Start recalculation of DCI values using preserver raw values.
+    * 
+    * @param objectId object ID
+    * @param dciId DCI ID
+    * @return assigned job ID
+    * @throws IOException  if socket I/O error occurs
+    * @throws NXCException if NetXMS server returns an error or operation was timed out
+    */
+   public long recalculateDCIValues(long objectId, long dciId) throws IOException, NXCException
+   {
+      final NXCPMessage msg = newMessage(NXCPCodes.CMD_RECALCULATE_DCI_VALUES);
+      msg.setFieldInt32(NXCPCodes.VID_OBJECT_ID, (int)objectId);
+      msg.setFieldInt32(NXCPCodes.VID_DCI_ID, (int)dciId);
+      sendMessage(msg);
+      NXCPMessage response = waitForRCC(msg.getMessageId());
+      return response.getFieldAsInt64(NXCPCodes.VID_JOB_ID);
+   }
+   
+   /**
     * Force DCI poll for given DCI
     *
     * @param nodeId Node object ID
index dfe9fae..fbe16a6 100644 (file)
                label="Force DCI Poll"
                menubarPath="secondary">
          </action>
+         <action
+               class="org.netxms.ui.eclipse.datacollection.actions.StartDataRecalculation"
+               enablesFor="+"
+               id="org.netxms.ui.eclipse.datacollection.popupActions.RecalculateData_DciValue"
+               label="Start data &amp;recalculation"
+               menubarPath="secondary">
+         </action>
       </objectContribution>
       <objectContribution
             adaptable="false"
                label="Force DCI Poll"
                menubarPath="secondary">
          </action>
+         <action
+               class="org.netxms.ui.eclipse.datacollection.actions.StartDataRecalculation"
+               enablesFor="+"
+               id="org.netxms.ui.eclipse.datacollection.popupActions.RecalculateData_DataCollectionObject"
+               label="Start data &amp;recalculation"
+               menubarPath="secondary">
+         </action>
       </objectContribution>
       <objectContribution
             adaptable="false"
diff --git a/src/java/netxms-eclipse/DataCollection/src/org/netxms/ui/eclipse/datacollection/actions/StartDataRecalculation.java b/src/java/netxms-eclipse/DataCollection/src/org/netxms/ui/eclipse/datacollection/actions/StartDataRecalculation.java
new file mode 100644 (file)
index 0000000..467437d
--- /dev/null
@@ -0,0 +1,173 @@
+/**
+ * NetXMS - open source network management system
+ * Copyright (C) 2003-2012 Victor Kirhenshtein
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.netxms.ui.eclipse.datacollection.actions;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.jface.action.IAction;
+import org.eclipse.jface.viewers.ISelection;
+import org.eclipse.jface.viewers.IStructuredSelection;
+import org.eclipse.ui.IObjectActionDelegate;
+import org.eclipse.ui.IWorkbenchPart;
+import org.netxms.client.NXCSession;
+import org.netxms.client.datacollection.DataCollectionObject;
+import org.netxms.client.datacollection.DciValue;
+import org.netxms.ui.eclipse.datacollection.Activator;
+import org.netxms.ui.eclipse.jobs.ConsoleJob;
+import org.netxms.ui.eclipse.shared.ConsoleSharedData;
+import org.netxms.ui.eclipse.tools.MessageDialogHelper;
+
+/**
+ * Clear collected data for DCI
+ */
+public class StartDataRecalculation implements IObjectActionDelegate
+{
+       private IWorkbenchPart part;
+       private Set<DCI> objects = null;
+
+       /* (non-Javadoc)
+        * @see org.eclipse.ui.IObjectActionDelegate#setActivePart(org.eclipse.jface.action.IAction, org.eclipse.ui.IWorkbenchPart)
+        */
+       @Override
+       public void setActivePart(IAction action, IWorkbenchPart targetPart)
+       {
+               part = targetPart;
+       }
+
+       /* (non-Javadoc)
+        * @see org.eclipse.ui.IActionDelegate#run(org.eclipse.jface.action.IAction)
+        */
+       @Override
+       public void run(IAction action)
+       {
+               if (objects == null)
+                       return;
+               
+               if (!MessageDialogHelper.openQuestion(part.getSite().getShell(), "Start Data Recalculation", "Collected values will be re-calculated using stored raw values and current transformation settings. Continue?"))
+                       return;
+               
+               final NXCSession session = (NXCSession)ConsoleSharedData.getSession();
+               final Set<DCI> dciToProcess = objects;
+               new ConsoleJob("Start DCI data recalculation", part, Activator.PLUGIN_ID, null) {
+                       @Override
+                       protected void runInternal(IProgressMonitor monitor) throws Exception
+                       {
+                               monitor.beginTask("Start DCI data recalculation", dciToProcess.size());
+                               for(DCI d : dciToProcess)
+                               {
+                                       session.recalculateDCIValues(d.nodeId, d.dciId);
+                                       monitor.worked(1);
+                               }
+                               monitor.done();
+                       }
+                       
+                       @Override
+                       protected String getErrorMessage()
+                       {
+                               return "Cannot initiate data recalculation";
+                       }
+               }.start();
+       }
+
+       /* (non-Javadoc)
+        * @see org.eclipse.ui.IActionDelegate#selectionChanged(org.eclipse.jface.action.IAction, org.eclipse.jface.viewers.ISelection)
+        */
+       @Override
+       public void selectionChanged(IAction action, ISelection selection)
+       {
+               Set<DCI> dciList = null;
+               if (selection instanceof IStructuredSelection)
+               {
+                       dciList = new HashSet<DCI>();
+                       for(Object o : ((IStructuredSelection)selection).toList())
+                       {
+                               if (o instanceof DciValue)
+                               {
+                                       dciList.add(new DCI(((DciValue)o).getNodeId(), ((DciValue)o).getId()));
+                               }
+                               else if (o instanceof DataCollectionObject)
+                               {
+                                       dciList.add(new DCI(((DataCollectionObject)o).getNodeId(), ((DataCollectionObject)o).getId()));
+                               }
+                               else
+                               {
+                                       dciList = null;
+                                       break;
+                               }
+                               
+                       }
+               }
+               
+               objects = dciList;
+               action.setEnabled((objects != null) && (objects.size() > 0));
+       }
+       
+       /**
+        *
+        */
+       private class DCI
+       {
+               long nodeId;
+               long dciId;
+               
+               /**
+                * @param nodeId
+                * @param dciId
+                */
+               public DCI(long nodeId, long dciId)
+               {
+                       this.nodeId = nodeId;
+                       this.dciId = dciId;
+               }
+
+               /* (non-Javadoc)
+                * @see java.lang.Object#hashCode()
+                */
+               @Override
+               public int hashCode()
+               {
+                       final int prime = 31;
+                       int result = 1;
+                       result = prime * result + (int)(dciId ^ (dciId >>> 32));
+                       result = prime * result + (int)(nodeId ^ (nodeId >>> 32));
+                       return result;
+               }
+               
+               /* (non-Javadoc)
+                * @see java.lang.Object#equals(java.lang.Object)
+                */
+               @Override
+               public boolean equals(Object obj)
+               {
+                       if (this == obj)
+                               return true;
+                       if (obj == null)
+                               return false;
+                       if (getClass() != obj.getClass())
+                               return false;
+                       DCI other = (DCI)obj;
+                       if (dciId != other.dciId)
+                               return false;
+                       if (nodeId != other.nodeId)
+                               return false;
+                       return true;
+               }
+       }
+}
index 5554230..ca7f440 100644 (file)
@@ -49,7 +49,7 @@ TCHAR LIBNETXMS_EXPORTABLE *NXCPMessageCodeName(WORD code, TCHAR *pszBuffer)
       _T("CMD_MODIFY_OBJECT"),
       _T("CMD_OBJECT_LIST_END"),
       _T("CMD_OBJECT_UPDATE"),
-      _T("<unused>0x000B"),
+      _T("CMD_RECALCULATE_DCI_VALUES"),
       _T("CMD_EVENTLOG_RECORDS"),
       _T("CMD_GET_CONFIG_VARLIST"),
       _T("CMD_SET_CONFIG_VARIABLE"),
index 805ed7d..9b08b5e 100644 (file)
@@ -34,7 +34,7 @@ public class NXCPCodes
        public static final int CMD_MODIFY_OBJECT = 0x0008;
        public static final int CMD_OBJECT_LIST_END = 0x0009;
        public static final int CMD_OBJECT_UPDATE = 0x000A;
-       //unused: public static final int CMD_GET_EVENTS = 0x000B;
+       public static final int CMD_RECALCULATE_DCI_VALUES = 0x000B;
        public static final int CMD_EVENTLOG_RECORDS = 0x000C;
        public static final int CMD_GET_CONFIG_VARLIST = 0x000D;
        public static final int CMD_SET_CONFIG_VARIABLE = 0x000E;
index 304f88f..3728790 100644 (file)
@@ -11,7 +11,7 @@ libnxcore_la_SOURCES =  accesspoint.cpp acl.cpp actions.cpp addrlist.cpp \
                        cert.cpp chassis.cpp client.cpp cluster.cpp columnfilter.cpp \
                        components.cpp condition.cpp config.cpp console.cpp \
                        container.cpp correlate.cpp dashboard.cpp datacoll.cpp dbwrite.cpp \
-                       dc_nxsl.cpp dcitem.cpp dcithreshold.cpp dcivalue.cpp \
+                       dc_nxsl.cpp dci_recalc.cpp dcitem.cpp dcithreshold.cpp dcivalue.cpp \
                        dcobject.cpp dcst.cpp dctable.cpp dctarget.cpp \
                        dctcolumn.cpp dctthreshold.cpp debug.cpp devdb.cpp dfile_info.cpp \
                        download_job.cpp ef.cpp email.cpp entirenet.cpp \
index 0895110..efe5f0c 100644 (file)
@@ -8,7 +8,7 @@ SOURCES =  accesspoint.cpp acl.cpp actions.cpp addrlist.cpp \
        cert.cpp chassis.cpp client.cpp cluster.cpp columnfilter.cpp \
        components.cpp condition.cpp config.cpp console.cpp \
        container.cpp correlate.cpp dashboard.cpp datacoll.cpp dbwrite.cpp \
-       dc_nxsl.cpp dcitem.cpp dcithreshold.cpp dcivalue.cpp \
+       dc_nxsl.cpp dci_recalc.cpp dcitem.cpp dcithreshold.cpp dcivalue.cpp \
        dcobject.cpp dcst.cpp dctable.cpp dctarget.cpp \
        dctcolumn.cpp dctthreshold.cpp debug.cpp devdb.cpp dfile_info.cpp \
        download_job.cpp ef.cpp email.cpp entirenet.cpp \
index 76b893c..0c925a3 100644 (file)
@@ -520,8 +520,8 @@ THREAD_RESULT THREAD_CALL CacheLoader(void *arg)
          DCObject *dci = static_cast<DataCollectionTarget*>(object)->getDCObjectById(ref->getId(), true);
          if ((dci != NULL) && (dci->getType() == DCO_TYPE_ITEM))
          {
-            DbgPrintf(6, _T("Loading cache for DCI %s [%d] on %s [%d]"),
-                      ref->getName(), ref->getId(), object->getName(), object->getId());
+            nxlog_debug_tag(_T("obj.dc.cache"), 6, _T("Loading cache for DCI %s [%d] on %s [%d]"),
+                     ref->getName(), ref->getId(), object->getName(), object->getId());
             static_cast<DCItem*>(dci)->reloadCache();
          }
          object->decRefCount();
diff --git a/src/server/core/dci_recalc.cpp b/src/server/core/dci_recalc.cpp
new file mode 100644 (file)
index 0000000..97dc462
--- /dev/null
@@ -0,0 +1,127 @@
+/*
+** NetXMS - Network Management System
+** Copyright (C) 2003-2017 Victor Kirhenshtein
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License as published by
+** the Free Software Foundation; either version 2 of the License, or
+** (at your option) any later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software
+** Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+**
+** File: dci_recalc.cpp
+**
+**/
+
+#include "nxcore.h"
+
+/**
+ * Recalculation job constructor
+ */
+DCIRecalculationJob::DCIRecalculationJob(DataCollectionTarget *object, DCItem *dci, UINT32 userId)
+                    : ServerJob(_T("DCI_RECALC"), _T("Recalculate DCI values"), object->getId(), userId, false)
+{
+   m_object = object;
+   m_object->incRefCount();
+   m_dci = new DCItem(dci);
+   m_cancelled = false;
+
+   TCHAR buffer[1024];
+   _sntprintf(buffer, 1024, _T("Recalculate values for DCI \"%s\" on %s"), dci->getDescription(), object->getName());
+   setDescription(buffer);
+}
+
+/**
+ * Recalculation job destructor
+ */
+DCIRecalculationJob::~DCIRecalculationJob()
+{
+   m_object->decRefCount();
+   delete m_dci;
+}
+
+/**
+ * Run job
+ */
+ServerJobResult DCIRecalculationJob::run()
+{
+   DB_HANDLE hdb = DBConnectionPoolAcquireConnection();
+
+   TCHAR query[256];
+   _sntprintf(query, 256, _T("SELECT idata_timestamp,raw_value FROM idata_%d WHERE item_id=%d ORDER BY idata_timestamp"), m_object->getId(), m_dci->getId());
+   DB_RESULT hResult = DBSelect(hdb, query);
+   if (hResult == NULL)
+   {
+      DBConnectionPoolReleaseConnection(hdb);
+      return JOB_RESULT_FAILED;
+   }
+
+   bool success = true;
+   int count = DBGetNumRows(hResult);
+   if (count > 0)
+   {
+      _sntprintf(query, 256, _T("UPDATE idata_%d SET idata_value=? WHERE item_id=? AND idata_timestamp=?"), m_object->getId());
+      DB_STATEMENT hStmt = DBPrepare(hdb, query);
+      if (hStmt != NULL)
+      {
+         m_dci->prepareForRecalc();
+         DBBegin(hdb);
+         for(int i = 0; (i < count) && !m_cancelled; i++)
+         {
+            time_t timestamp = static_cast<time_t>(DBGetFieldInt64(hResult, i, 0));
+            TCHAR data[MAX_RESULT_LENGTH];
+            DBGetField(hResult, i, 1, data, MAX_RESULT_LENGTH);
+            ItemValue value(data, timestamp);
+            m_dci->recalculateValue(value);
+
+            DBBind(hStmt, 1, DB_SQLTYPE_VARCHAR, value.getString(), DB_BIND_STATIC);
+            DBBind(hStmt, 2, DB_SQLTYPE_INTEGER, m_dci->getId());
+            DBBind(hStmt, 3, DB_SQLTYPE_INTEGER, (UINT32)value.getTimeStamp());
+            DBExecute(hStmt);
+
+            if (i % 10 == 0)
+            {
+               markProgress(i * 100 / count);
+            }
+
+            if (i % 1000 == 0)
+            {
+               DBCommit(hdb);
+               DBBegin(hdb);
+            }
+         }
+         DBCommit(hdb);
+         DBFreeStatement(hStmt);
+      }
+      else
+      {
+         success = false;
+      }
+   }
+
+   DBFreeResult(hResult);
+   DBConnectionPoolReleaseConnection(hdb);
+
+   if (success)
+   {
+      m_object->reloadDCItemCache(m_dci->getId());
+      markProgress(100);
+   }
+   return success ? JOB_RESULT_SUCCESS : JOB_RESULT_FAILED;
+}
+
+/**
+ * Cancel job
+ */
+bool DCIRecalculationJob::onCancel()
+{
+   m_cancelled = true;
+   return true;
+}
index f4cb8c0..c12597d 100644 (file)
@@ -191,7 +191,7 @@ DCItem::DCItem(UINT32 dwId, const TCHAR *szName, int iSource, int iDataType,
        m_snmpRawValueType = SNMP_RAWTYPE_NONE;
        m_predictionEngine[0] = 0;
 
-   updateCacheSizeInternal();
+   updateCacheSizeInternal(false);
 }
 
 /**
@@ -235,7 +235,7 @@ DCItem::DCItem(ConfigEntry *config, Template *owner) : DCObject(config, owner)
                m_thresholds = NULL;
        }
 
-       updateCacheSizeInternal();
+       updateCacheSizeInternal(false);
 }
 
 /**
@@ -645,9 +645,9 @@ void DCItem::updateFromMessage(NXCPMessage *pMsg, UINT32 *pdwNumMaps, UINT32 **p
    for(int i = 0; i < getThresholdCount(); i++)
       m_thresholds->get(i)->setDataType(m_dataType);
 
-       safe_free(ppNewList);
-   safe_free(newThresholds);
-   updateCacheSizeInternal();
+       free(ppNewList);
+   free(newThresholds);
+   updateCacheSizeInternal(true);
    unlock();
 }
 
@@ -987,7 +987,7 @@ void DCItem::changeBinding(UINT32 dwNewId, Template *pNewNode, BOOL doMacroExpan
        }
 
    clearCache();
-   updateCacheSizeInternal();
+   updateCacheSizeInternal(true);
    unlock();
 }
 
@@ -997,10 +997,8 @@ void DCItem::changeBinding(UINT32 dwNewId, Template *pNewNode, BOOL doMacroExpan
  * GetCacheSizeForDCI should be called with bNoLock == TRUE for appropriate
  * condition object
  */
-void DCItem::updateCacheSizeInternal(UINT32 conditionId)
+void DCItem::updateCacheSizeInternal(bool allowLoad, UINT32 conditionId)
 {
-   UINT32 dwSize, dwRequiredSize;
-
    // Sanity check
    if (m_owner == NULL)
    {
@@ -1014,41 +1012,45 @@ void DCItem::updateCacheSizeInternal(UINT32 conditionId)
         ((m_owner->getObjectClass() == OBJECT_CLUSTER) && isAggregateOnCluster())) &&
        (m_instanceDiscoveryMethod == IDM_NONE))
    {
-      dwRequiredSize = 1;
+      UINT32 requiredSize = 1;
 
       // Calculate required cache size
       for(int i = 0; i < getThresholdCount(); i++)
-         if (dwRequiredSize < m_thresholds->get(i)->getRequiredCacheSize())
-            dwRequiredSize = m_thresholds->get(i)->getRequiredCacheSize();
+         if (requiredSize < m_thresholds->get(i)->getRequiredCacheSize())
+            requiredSize = m_thresholds->get(i)->getRequiredCacheSize();
 
                ObjectArray<NetObj> *conditions = g_idxConditionById.getObjects(true);
                for(int i = 0; i < conditions->size(); i++)
       {
                   ConditionObject *c = (ConditionObject *)conditions->get(i);
-                       dwSize = c->getCacheSizeForDCI(m_id, conditionId == c->getId());
-         if (dwSize > dwRequiredSize)
-            dwRequiredSize = dwSize;
+                       UINT32 size = c->getCacheSizeForDCI(m_id, conditionId == c->getId());
+         if (size > requiredSize)
+            requiredSize = size;
          c->decRefCount();
       }
                delete conditions;
+
+               m_requiredCacheSize = requiredSize;
    }
    else
    {
-      dwRequiredSize = 0;
+      m_requiredCacheSize = 0;
    }
 
+   nxlog_debug_tag(_T("obj.dc.cache"), 8, _T("DCItem::updateCacheSizeInternal(dci=\"%s\", node=%s [%d]): requiredSize=%d cacheSize=%d"),
+            m_name, m_owner->getName(), m_owner->getId(), m_requiredCacheSize, m_cacheSize);
+
    // Update cache if needed
-   if (dwRequiredSize < m_cacheSize)
+   if (m_requiredCacheSize < m_cacheSize)
    {
       // Destroy unneeded values
       if (m_cacheSize > 0)
                {
-         for(UINT32 i = dwRequiredSize; i < m_cacheSize; i++)
+         for(UINT32 i = m_requiredCacheSize; i < m_cacheSize; i++)
             delete m_ppValueCache[i];
                }
 
-      m_cacheSize = dwRequiredSize;
-      m_requiredCacheSize = dwRequiredSize;
+      m_cacheSize = m_requiredCacheSize;
       if (m_cacheSize > 0)
       {
          m_ppValueCache = (ItemValue **)realloc(m_ppValueCache, sizeof(ItemValue *) * m_cacheSize);
@@ -1059,25 +1061,24 @@ void DCItem::updateCacheSizeInternal(UINT32 conditionId)
          m_ppValueCache = NULL;
       }
    }
-   else if (dwRequiredSize > m_cacheSize)
+   else if (m_requiredCacheSize > m_cacheSize)
    {
       // Load missing values from database
       // Skip caching for DCIs where estimated time to fill the cache is less then 5 minutes
       // to reduce load on database at server startup
-      if ((m_owner != NULL) && (((dwRequiredSize - m_cacheSize) * m_iPollingInterval > 300) || (m_source == DS_PUSH_AGENT)))
+      if (allowLoad && (m_owner != NULL) && (((m_requiredCacheSize - m_cacheSize) * m_iPollingInterval > 300) || (m_source == DS_PUSH_AGENT)))
       {
-         m_requiredCacheSize = dwRequiredSize;
          m_bCacheLoaded = false;
          g_dciCacheLoaderQueue.put(new DCObjectInfo(this));
       }
       else
       {
          // will not read data from database, fill cache with empty values
-         m_ppValueCache = (ItemValue **)realloc(m_ppValueCache, sizeof(ItemValue *) * dwRequiredSize);
-         for(UINT32 i = m_cacheSize; i < dwRequiredSize; i++)
+         m_ppValueCache = (ItemValue **)realloc(m_ppValueCache, sizeof(ItemValue *) * m_requiredCacheSize);
+         for(UINT32 i = m_cacheSize; i < m_requiredCacheSize; i++)
             m_ppValueCache[i] = new ItemValue(_T(""), 1);
          DbgPrintf(7, _T("Cache load skipped for parameter %s [%d]"), m_name, (int)m_id);
-         m_cacheSize = dwRequiredSize;
+         m_cacheSize = m_requiredCacheSize;
          m_bCacheLoaded = true;
       }
    }
@@ -1135,6 +1136,8 @@ void DCItem::reloadCache()
       m_ppValueCache = (ItemValue **)realloc(m_ppValueCache, sizeof(ItemValue *) * m_requiredCacheSize);
    }
 
+   nxlog_debug_tag(_T("obj.dc.cache"), 8, _T("DCItem::reloadCache(dci=\"%s\", node=%s [%d]): requiredSize=%d cacheSize=%d"),
+            m_name, m_owner->getName(), m_owner->getId(), m_requiredCacheSize, m_cacheSize);
    if (hResult != NULL)
    {
       // Create cache entries
@@ -1154,9 +1157,13 @@ void DCItem::reloadCache()
       }
 
       // Fill up cache with empty values if we don't have enough values in database
-      for(; i < m_requiredCacheSize; i++)
-         m_ppValueCache[i] = new ItemValue(_T(""), 1);
-
+      if (i < m_requiredCacheSize)
+      {
+         nxlog_debug_tag(_T("obj.dc.cache"), 8, _T("DCItem::reloadCache(dci=\"%s\", node=%s [%d]): %d values missing in DB"),
+                  m_name, m_owner->getName(), m_owner->getId(), m_requiredCacheSize - i);
+         for(; i < m_requiredCacheSize; i++)
+            m_ppValueCache[i] = new ItemValue(_T(""), 1);
+      }
       DBFreeResult(hResult);
    }
    else
@@ -1387,7 +1394,7 @@ bool DCItem::deleteAllData()
        success = DBQuery(hdb, szQuery) ? true : false;
    DBConnectionPoolReleaseConnection(hdb);
        clearCache();
-       updateCacheSizeInternal();
+       updateCacheSizeInternal(true);
    unlock();
        return success;
 }
@@ -1447,7 +1454,7 @@ void DCItem::updateFromTemplate(DCObject *src)
    for(i = 0; i < getThresholdCount(); i++)
       m_thresholds->get(i)->setDataType(m_dataType);
 
-   updateCacheSizeInternal();
+   updateCacheSizeInternal(true);
    unlock();
 }
 
@@ -1738,7 +1745,7 @@ void DCItem::updateFromImport(ConfigEntry *config)
       delete_and_null(m_thresholds);
    }
 
-   updateCacheSizeInternal();
+   updateCacheSizeInternal(true);
    unlock();
 }
 
@@ -1769,3 +1776,48 @@ json_t *DCItem::toJson()
    json_object_set_new(root, "predictionEngine", json_string_t(m_predictionEngine));
    return root;
 }
+
+/**
+ * Prepare DCI object for recalculation (should be executed on DCI copy)
+ */
+void DCItem::prepareForRecalc()
+{
+   m_tPrevValueTimeStamp = 0;
+   m_tLastPoll = 0;
+   updateCacheSizeInternal(false);
+}
+
+/**
+ * Recalculate old value (should be executed on DCI copy)
+ */
+void DCItem::recalculateValue(ItemValue &value)
+{
+   if (m_tPrevValueTimeStamp == 0)
+      m_prevRawValue = value;  // Delta should be zero for first poll
+   ItemValue rawValue = value;
+
+   // Cluster can have only aggregated data, and transformation
+   // should not be used on aggregation
+   if ((m_owner->getObjectClass() != OBJECT_CLUSTER) || (m_flags & DCF_TRANSFORM_AGGREGATED))
+   {
+      if (!transform(value, (value.getTimeStamp() > m_tPrevValueTimeStamp) ? (value.getTimeStamp() - m_tPrevValueTimeStamp) : 0))
+      {
+         return;
+      }
+   }
+
+   if (value.getTimeStamp() > m_tPrevValueTimeStamp)
+   {
+      m_prevRawValue = rawValue;
+      m_tPrevValueTimeStamp = value.getTimeStamp();
+   }
+
+   if ((m_cacheSize > 0) && (value.getTimeStamp() >= m_tPrevValueTimeStamp))
+   {
+      delete m_ppValueCache[m_cacheSize - 1];
+      memmove(&m_ppValueCache[1], m_ppValueCache, sizeof(ItemValue *) * (m_cacheSize - 1));
+      m_ppValueCache[0] = new ItemValue(&value);
+   }
+
+   m_tLastPoll = value.getTimeStamp();
+}
index a973cb1..ff6c824 100644 (file)
@@ -91,7 +91,7 @@ DCObject::DCObject(const DCObject *pSrc)
        _tcscpy(m_name, pSrc->m_name);
        _tcscpy(m_description, pSrc->m_description);
        _tcscpy(m_systemTag, pSrc->m_systemTag);
-   m_owner = NULL;
+   m_owner = pSrc->m_owner;
    m_hMutex = MutexCreateRecursive();
    m_tLastCheck = 0;
    m_dwErrorCount = 0;
index b3a4a32..fb569b0 100644 (file)
@@ -1159,7 +1159,23 @@ void DataCollectionTarget::updateDCItemCacheSize(UINT32 dciId, UINT32 conditionI
    DCObject *dci = getDCObjectById(dciId, false);
    if ((dci != NULL) && (dci->getType() == DCO_TYPE_ITEM))
    {
-      ((DCItem *)dci)->updateCacheSize(conditionId);
+      static_cast<DCItem*>(dci)->updateCacheSize(conditionId);
+   }
+   unlockDciAccess();
+}
+
+/**
+ * Reload DCI cache
+ */
+void DataCollectionTarget::reloadDCItemCache(UINT32 dciId)
+{
+   lockDciAccess(false);
+   DCObject *dci = getDCObjectById(dciId, false);
+   if ((dci != NULL) && (dci->getType() == DCO_TYPE_ITEM))
+   {
+      nxlog_debug_tag(_T("obj.dc.cache"), 6, _T("Reload DCI cache for \"%s\" [%d] on %s [%d]"),
+               dci->getName(), dci->getId(), m_name, m_id);
+      static_cast<DCItem*>(dci)->reloadCache();
    }
    unlockDciAccess();
 }
index 6b8c683..9b4bfb7 100644 (file)
@@ -167,6 +167,7 @@ DEFINE_THREAD_STARTER(queryAgentTable)
 DEFINE_THREAD_STARTER(queryL2Topology)
 DEFINE_THREAD_STARTER(queryParameter)
 DEFINE_THREAD_STARTER(queryServerLog)
+DEFINE_THREAD_STARTER(recalculateDCIValues)
 DEFINE_THREAD_STARTER(sendMib)
 DEFINE_THREAD_STARTER(uploadUserFileToAgent)
 DEFINE_THREAD_STARTER(getRepositories)
@@ -768,6 +769,9 @@ void ClientSession::processingThread()
                        case CMD_FORCE_DCI_POLL:
                                CALL_IN_NEW_THREAD(forceDCIPoll, pMsg);
                                break;
+                       case CMD_RECALCULATE_DCI_VALUES:
+            CALL_IN_NEW_THREAD(recalculateDCIValues, pMsg);
+                          break;
          case CMD_OPEN_EPP:
             openEPP(pMsg);
             break;
@@ -3531,42 +3535,105 @@ void ClientSession::changeDCIStatus(NXCPMessage *request)
 }
 
 /**
+ * Recalculate values for DCI
+ */
+void ClientSession::recalculateDCIValues(NXCPMessage *request)
+{
+   NXCPMessage msg(CMD_REQUEST_COMPLETED, request->getId());
+
+   NetObj *object = FindObjectById(request->getFieldAsUInt32(VID_OBJECT_ID));
+   if (object != NULL)
+   {
+      if (object->isDataCollectionTarget())
+      {
+         if (object->checkAccessRights(m_dwUserId, OBJECT_ACCESS_MODIFY))
+         {
+            UINT32 dciId = request->getFieldAsUInt32(VID_DCI_ID);
+            debugPrintf(4, _T("recalculateDCIValues: request for DCI %d at target %s [%d]"), dciId, object->getName(), object->getId());
+            DCObject *dci = static_cast<DataCollectionTarget*>(object)->getDCObjectById(dciId);
+            if (dci != NULL)
+            {
+               if (dci->getType() == DCO_TYPE_ITEM)
+               {
+                  debugPrintf(4, _T("recalculateDCIValues: DCI \"%s\" [%d] at target %s [%d]"), dci->getDescription(), dciId, object->getName(), object->getId());
+                  DCIRecalculationJob *job = new DCIRecalculationJob(static_cast<DataCollectionTarget*>(object), static_cast<DCItem*>(dci), m_dwUserId);
+                  if (AddJob(job))
+                  {
+                     msg.setField(VID_RCC, RCC_SUCCESS);
+                     msg.setField(VID_JOB_ID, job->getId());
+                     writeAuditLog(AUDIT_OBJECTS, true, object->getId(), _T("Data recalculation for DCI \"%s\" [%d] on object \"%s\" [%d] started (job ID %d)"),
+                              dci->getDescription(), dci->getId(), object->getName(), object->getId(), job->getId());
+                  }
+                  else
+                  {
+                     delete job;
+                     msg.setField(VID_RCC, RCC_INTERNAL_ERROR);
+                  }
+               }
+               else
+               {
+                  msg.setField(VID_RCC, RCC_INCOMPATIBLE_OPERATION);
+               }
+            }
+            else
+            {
+               msg.setField(VID_RCC, RCC_INVALID_DCI_ID);
+               debugPrintf(4, _T("recalculateDCIValues: DCI %d at target %s [%d] not found"), dciId, object->getName(), object->getId());
+            }
+         }
+         else  // User doesn't have MODIFY rights on object
+         {
+            msg.setField(VID_RCC, RCC_ACCESS_DENIED);
+            writeAuditLog(AUDIT_OBJECTS, false, object->getId(), _T("Access denied on recalculating DCI data"));
+         }
+      }
+      else     // Object is not a node
+      {
+         msg.setField(VID_RCC, RCC_INCOMPATIBLE_OPERATION);
+      }
+   }
+   else  // No object with given ID
+   {
+      msg.setField(VID_RCC, RCC_INVALID_OBJECT_ID);
+   }
+
+   sendMessage(&msg);
+}
+
+/**
  * Clear all collected data for DCI
  */
 void ClientSession::clearDCIData(NXCPMessage *request)
 {
-   NXCPMessage msg;
-   NetObj *object;
-       UINT32 dwItemId;
-
-   // Prepare response message
-   msg.setCode(CMD_REQUEST_COMPLETED);
-   msg.setId(request->getId());
+   NXCPMessage msg(CMD_REQUEST_COMPLETED, request->getId());
 
    // Get node id and check object class and access rights
-   object = FindObjectById(request->getFieldAsUInt32(VID_OBJECT_ID));
+   NetObj *object = FindObjectById(request->getFieldAsUInt32(VID_OBJECT_ID));
    if (object != NULL)
    {
       if (object->isDataCollectionTarget())
       {
          if (object->checkAccessRights(m_dwUserId, OBJECT_ACCESS_DELETE))
          {
-                               dwItemId = request->getFieldAsUInt32(VID_DCI_ID);
-                               debugPrintf(4, _T("ClearDCIData: request for DCI %d at node %d"), dwItemId, object->getId());
-            DCObject *dci = ((Template *)object)->getDCObjectById(dwItemId);
+                               UINT32 dciId = request->getFieldAsUInt32(VID_DCI_ID);
+                               debugPrintf(4, _T("ClearDCIData: request for DCI %d at node %d"), dciId, object->getId());
+            DCObject *dci = ((Template *)object)->getDCObjectById(dciId);
                                if (dci != NULL)
                                {
                                        msg.setField(VID_RCC, dci->deleteAllData() ? RCC_SUCCESS : RCC_DB_FAILURE);
-                                       debugPrintf(4, _T("ClearDCIData: DCI %d at node %d"), dwItemId, object->getId());
+                                       debugPrintf(4, _T("ClearDCIData: DCI %d at node %d"), dciId, object->getId());
+                   writeAuditLog(AUDIT_OBJECTS, true, object->getId(), _T("Collected data for DCI \"%s\" [%d] on object \"%s\" [%d] cleared"),
+                            dci->getDescription(), dci->getId(), object->getName(), object->getId());
                                }
                                else
                                {
                                        msg.setField(VID_RCC, RCC_INVALID_DCI_ID);
-                                       debugPrintf(4, _T("ClearDCIData: DCI %d at node %d not found"), dwItemId, object->getId());
+                                       debugPrintf(4, _T("ClearDCIData: DCI %d at node %d not found"), dciId, object->getId());
                                }
          }
          else  // User doesn't have DELETE rights on object
          {
+            writeAuditLog(AUDIT_OBJECTS, false, object->getId(), _T("Access denied on clear DCI data"));
             msg.setField(VID_RCC, RCC_ACCESS_DENIED);
          }
       }
index 1c7503b..7d53e22 100644 (file)
@@ -499,6 +499,7 @@ private:
    DECLARE_THREAD_STARTER(queryL2Topology)
    DECLARE_THREAD_STARTER(queryParameter)
    DECLARE_THREAD_STARTER(queryServerLog)
+   DECLARE_THREAD_STARTER(recalculateDCIValues)
    DECLARE_THREAD_STARTER(sendMib)
    DECLARE_THREAD_STARTER(uploadUserFileToAgent)
    DECLARE_THREAD_STARTER(getRepositories)
@@ -554,6 +555,7 @@ private:
        bool getCollectedDataFromDB(NXCPMessage *request, NXCPMessage *response, DataCollectionTarget *object, int dciType, bool withRawValues);
        void clearDCIData(NXCPMessage *pRequest);
        void forceDCIPoll(NXCPMessage *pRequest);
+   void recalculateDCIValues(NXCPMessage *request);
    void changeDCIStatus(NXCPMessage *pRequest);
    void getLastValues(NXCPMessage *pRequest);
    void getLastValuesByDciId(NXCPMessage *pRequest);
index 7588db2..e3aeba7 100644 (file)
@@ -360,7 +360,7 @@ protected:
 
    bool transform(ItemValue &value, time_t nElapsedTime);
    void checkThresholds(ItemValue &value);
-   void updateCacheSizeInternal(UINT32 conditionId = 0);
+   void updateCacheSizeInternal(bool allowLoad, UINT32 conditionId = 0);
    void clearCache();
 
        virtual bool isCacheLoaded();
@@ -388,7 +388,7 @@ public:
    virtual void deleteFromDatabase();
    virtual bool loadThresholdsFromDB(DB_HANDLE hdb);
 
-   void updateCacheSize(UINT32 conditionId = 0) { lock(); updateCacheSizeInternal(conditionId); unlock(); }
+   void updateCacheSize(UINT32 conditionId = 0) { lock(); updateCacheSizeInternal(true, conditionId); unlock(); }
    void reloadCache();
 
    int getDataType() const { return m_dataType; }
@@ -430,6 +430,9 @@ public:
        void addThreshold(Threshold *pThreshold);
        void deleteAllThresholds();
 
+       void prepareForRecalc();
+       void recalculateValue(ItemValue &value);
+
    static bool testTransformation(DataCollectionTarget *object, const TCHAR *script, const TCHAR *value, TCHAR *buffer, size_t bufSize);
 };
 
index fa539fe..7835638 100644 (file)
@@ -1136,6 +1136,7 @@ public:
 
    void updateDciCache();
    void updateDCItemCacheSize(UINT32 dciId, UINT32 conditionId = 0);
+   void reloadDCItemCache(UINT32 dciId);
    void cleanDCIData(DB_HANDLE hdb);
    void queueItemsForPolling();
        bool processNewDCValue(DCObject *dco, time_t currTime, const void *value);
index aab1d47..772f70f 100644 (file)
@@ -245,6 +245,25 @@ public:
        const TCHAR *getLocalFileName();
 };
 
+/**
+ * DCI recalculation job
+ */
+class DCIRecalculationJob : public ServerJob
+{
+private:
+   DataCollectionTarget *m_object;
+   DCItem *m_dci;
+   bool m_cancelled;
+
+protected:
+   virtual ServerJobResult run();
+   virtual bool onCancel();
+
+public:
+   DCIRecalculationJob(DataCollectionTarget *object, DCItem *dci, UINT32 userId);
+   virtual ~DCIRecalculationJob();
+};
+
 class AgentPolicy;
 
 /**
index dfe9fae..fbe16a6 100644 (file)
                label="Force DCI Poll"
                menubarPath="secondary">
          </action>
+         <action
+               class="org.netxms.ui.eclipse.datacollection.actions.StartDataRecalculation"
+               enablesFor="+"
+               id="org.netxms.ui.eclipse.datacollection.popupActions.RecalculateData_DciValue"
+               label="Start data &amp;recalculation"
+               menubarPath="secondary">
+         </action>
       </objectContribution>
       <objectContribution
             adaptable="false"
                label="Force DCI Poll"
                menubarPath="secondary">
          </action>
+         <action
+               class="org.netxms.ui.eclipse.datacollection.actions.StartDataRecalculation"
+               enablesFor="+"
+               id="org.netxms.ui.eclipse.datacollection.popupActions.RecalculateData_DataCollectionObject"
+               label="Start data &amp;recalculation"
+               menubarPath="secondary">
+         </action>
       </objectContribution>
       <objectContribution
             adaptable="false"
diff --git a/webui/webapp/DataCollection/src/org/netxms/ui/eclipse/datacollection/actions/StartDataRecalculation.java b/webui/webapp/DataCollection/src/org/netxms/ui/eclipse/datacollection/actions/StartDataRecalculation.java
new file mode 100644 (file)
index 0000000..467437d
--- /dev/null
@@ -0,0 +1,173 @@
+/**
+ * NetXMS - open source network management system
+ * Copyright (C) 2003-2012 Victor Kirhenshtein
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.netxms.ui.eclipse.datacollection.actions;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.jface.action.IAction;
+import org.eclipse.jface.viewers.ISelection;
+import org.eclipse.jface.viewers.IStructuredSelection;
+import org.eclipse.ui.IObjectActionDelegate;
+import org.eclipse.ui.IWorkbenchPart;
+import org.netxms.client.NXCSession;
+import org.netxms.client.datacollection.DataCollectionObject;
+import org.netxms.client.datacollection.DciValue;
+import org.netxms.ui.eclipse.datacollection.Activator;
+import org.netxms.ui.eclipse.jobs.ConsoleJob;
+import org.netxms.ui.eclipse.shared.ConsoleSharedData;
+import org.netxms.ui.eclipse.tools.MessageDialogHelper;
+
+/**
+ * Clear collected data for DCI
+ */
+public class StartDataRecalculation implements IObjectActionDelegate
+{
+       private IWorkbenchPart part;
+       private Set<DCI> objects = null;
+
+       /* (non-Javadoc)
+        * @see org.eclipse.ui.IObjectActionDelegate#setActivePart(org.eclipse.jface.action.IAction, org.eclipse.ui.IWorkbenchPart)
+        */
+       @Override
+       public void setActivePart(IAction action, IWorkbenchPart targetPart)
+       {
+               part = targetPart;
+       }
+
+       /* (non-Javadoc)
+        * @see org.eclipse.ui.IActionDelegate#run(org.eclipse.jface.action.IAction)
+        */
+       @Override
+       public void run(IAction action)
+       {
+               if (objects == null)
+                       return;
+               
+               if (!MessageDialogHelper.openQuestion(part.getSite().getShell(), "Start Data Recalculation", "Collected values will be re-calculated using stored raw values and current transformation settings. Continue?"))
+                       return;
+               
+               final NXCSession session = (NXCSession)ConsoleSharedData.getSession();
+               final Set<DCI> dciToProcess = objects;
+               new ConsoleJob("Start DCI data recalculation", part, Activator.PLUGIN_ID, null) {
+                       @Override
+                       protected void runInternal(IProgressMonitor monitor) throws Exception
+                       {
+                               monitor.beginTask("Start DCI data recalculation", dciToProcess.size());
+                               for(DCI d : dciToProcess)
+                               {
+                                       session.recalculateDCIValues(d.nodeId, d.dciId);
+                                       monitor.worked(1);
+                               }
+                               monitor.done();
+                       }
+                       
+                       @Override
+                       protected String getErrorMessage()
+                       {
+                               return "Cannot initiate data recalculation";
+                       }
+               }.start();
+       }
+
+       /* (non-Javadoc)
+        * @see org.eclipse.ui.IActionDelegate#selectionChanged(org.eclipse.jface.action.IAction, org.eclipse.jface.viewers.ISelection)
+        */
+       @Override
+       public void selectionChanged(IAction action, ISelection selection)
+       {
+               Set<DCI> dciList = null;
+               if (selection instanceof IStructuredSelection)
+               {
+                       dciList = new HashSet<DCI>();
+                       for(Object o : ((IStructuredSelection)selection).toList())
+                       {
+                               if (o instanceof DciValue)
+                               {
+                                       dciList.add(new DCI(((DciValue)o).getNodeId(), ((DciValue)o).getId()));
+                               }
+                               else if (o instanceof DataCollectionObject)
+                               {
+                                       dciList.add(new DCI(((DataCollectionObject)o).getNodeId(), ((DataCollectionObject)o).getId()));
+                               }
+                               else
+                               {
+                                       dciList = null;
+                                       break;
+                               }
+                               
+                       }
+               }
+               
+               objects = dciList;
+               action.setEnabled((objects != null) && (objects.size() > 0));
+       }
+       
+       /**
+        *
+        */
+       private class DCI
+       {
+               long nodeId;
+               long dciId;
+               
+               /**
+                * @param nodeId
+                * @param dciId
+                */
+               public DCI(long nodeId, long dciId)
+               {
+                       this.nodeId = nodeId;
+                       this.dciId = dciId;
+               }
+
+               /* (non-Javadoc)
+                * @see java.lang.Object#hashCode()
+                */
+               @Override
+               public int hashCode()
+               {
+                       final int prime = 31;
+                       int result = 1;
+                       result = prime * result + (int)(dciId ^ (dciId >>> 32));
+                       result = prime * result + (int)(nodeId ^ (nodeId >>> 32));
+                       return result;
+               }
+               
+               /* (non-Javadoc)
+                * @see java.lang.Object#equals(java.lang.Object)
+                */
+               @Override
+               public boolean equals(Object obj)
+               {
+                       if (this == obj)
+                               return true;
+                       if (obj == null)
+                               return false;
+                       if (getClass() != obj.getClass())
+                               return false;
+                       DCI other = (DCI)obj;
+                       if (dciId != other.dciId)
+                               return false;
+                       if (nodeId != other.nodeId)
+                               return false;
+                       return true;
+               }
+       }
+}