Source

eyes / eyeswebapp / core / models.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
"""
Eyes Core Models
================

* models supporting the core monitoring infrastructure

"""
import datetime
import dateutil.parser
import os
import simplejson
import tagging

from django.db import models
from settings import RRDFILE_ROOT
from settings import PNGFILE_ROOT
from tagging.fields import TagField
from pyrrd.rrd import RRD, RRA, DS
from pyrrd.graph import DEF, VDEF  # , CDEF
from pyrrd.graph import LINE, AREA  # , GPRINT
from pyrrd.graph import ColorAttributes, Graph
from pyrrd.util import epoch
from util.monitor import ArgSet
from util.monitor import validate_poller_results
from util.monitor import validate_return_dictionary


class Host(models.Model):
    """ class representing a host. A host may have 0 or more associated monitors.

    >>> from core.models import Host
    >>> x = Host()
    >>> x.hostname="localhost"
    >>> x.save()
    >>> x
    <Host: localhost>
    >>> x.monitor_set.count()
    0

    """
    # pylint: disable=E1101
    hostname = models.CharField(max_length=250, default="localhost")
    tags_string = TagField()
        # Don't name this tags as it will conflict with the tags attribute gained
        # through registering the model with tagging

    class Meta:
        """META class for Django model administration - basic ordering, naming."""
        ordering = ['hostname']
        verbose_name, verbose_name_plural = "Host", "Hosts"

    def __unicode__(self):
        """unicode string representation of a Monitor"""
        return u"%s" % (self.hostname, )

    @models.permalink
    def get_absolute_url(self):
        """ returns the absolute URL for the monitor element"""
        return ('core.views.host_detail', [str(self.id)])

try:
    tagging.register(Host)
except tagging.AlreadyRegistered:
    # You might wonder what is going on here.
    # This is a bug in the svn release.  For some reason, the authors' cannot handle
    # re-registering gracefully, so we have to.
    # This should be fixed in later releases, but your code will still work here after it is.
    pass


class MonitorManager(models.Manager):
    """ manager object for manipulating monitor elements"""
    # pylint: disable=E1101
    def pending_update(self):
        """ returns the iterable django query set containing monitor objects that are due for updating.
        i.e. Monitor.objects.pending_update()
        """
        basic_set = self.get_query_set().exclude(passive__exact=True).exclude(nextupdate__gte=datetime.datetime.now())
        return basic_set


class Monitor(models.Model):
    """A monitor in the Eyes system

    >>> from core.models import Monitor
    >>> from util.monitor import ArgSet
    >>> x = Monitor()
    >>> x.name="selfping"
    >>> x.plugin_name="check_ping"
    >>> args = ArgSet()
    >>> args.add_argument_pair("-H", "localhost")
    >>> args.add_argument_pair("-w", "1,99%")
    >>> args.add_argument_pair("-c", "1,99%")
    >>> x.arg_set = args
    >>> x.save()
    >>> x
    <Monitor: Monitor selfping (check_ping) against None>
    >>> x.json_argset
    '["-H localhost", "-w 1,99%", "-c 1,99%"]'

    """
    # pylint: disable=E1101
    STATE_CHOICES = (
        (0, u'ok'),  # eyes_happy
        (1, u'warning'),  # eyes_worry
        (2, u'error'),  # eyes_angry
        (3, u'unknown'),  # eyes_wideopen
    )

    objects = MonitorManager()
    #
    name = models.CharField(max_length=250, default='')  # user specified name
    plugin_name = models.CharField(max_length=250, default="check_ping")  # names of nagios plugins - like check_snmp
    json_argset = models.TextField(blank=True)  # json representation of an ArgSet
    #
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now_add=True, default=datetime.datetime.now)
    #
    poll_frequency = models.IntegerField(default=5)  # value in minutes
    lastupdate = models.DateTimeField(null=True)  # to be queried to determine what needs to be updated
    nextupdate = models.DateTimeField(null=True, db_index=True)  # denormalized data to make querying against a narrower gap more effective - to be set on each update based on freq requested
    # state information - denormalized into this immediate state and previous run list as a sep. model?
    latest_state = models.SmallIntegerField(default=0, choices=STATE_CHOICES)
    latest_result_string = models.CharField(max_length=255, blank=True)
    #
    alerting = models.BooleanField(default=True)
    passive = models.BooleanField(default=False, db_index=True)  # identifies a passive monitor
    #
    host = models.ForeignKey(Host, null=True)
    tags_string = TagField()
        # Don't name this tags as it will conflict with the tags attribute gained
        # through registering the model with tagging

    # reconfigured to use python properties to return an util.monitor.ArgSet object from JSON data store
    def _get_arg_set(self):
        """ returns an argument set (ArgSet) object for a given monitor, None by default"""
        if self.json_argset is None:
            return None
        if self.json_argset == '':
            return None
        new_arg_set = ArgSet()
        new_arg_set.loadjson(self.json_argset)
        return new_arg_set

    def _set_arg_set(self, arg_set):
        """ takes an ArgSet object (NagiosInvoker argument set) and stores it into the monitor
        as the ArgSet's JSON representation"""
        if (arg_set is None):
            self.json_argset = ''
        else:
            self.json_argset = arg_set.json()
    arg_set = property(_get_arg_set, _set_arg_set)

    def _get_state(self):
        """ returns the latest state of the monitor """
        return self.latest_state

    def _set_state(self, incoming_state):
        """ updates the lastupdate and nextupdate timestamp on the object.
        (0, u'ok') -- state is OK according to threshold or monitor
        (1, u'warning') -- warning: not OK, but not error either
        (2, u'error') -- something's gone very off expectations
        (3, u'unknown') -- used for monitors that just collect metrics without analysis, or prior to any collection
        """
        now = datetime.datetime.now()
        self.lastupdate = now
        if not(self.passive):
            self.nextupdate = now + datetime.timedelta(minutes=self.poll_frequency)
        if (incoming_state >= 0) and (incoming_state < 4):
            self.latest_state = incoming_state
    state = property(_get_state, _set_state)

    def _get_state_display(self):
        for key, strvalue in Monitor.STATE_CHOICES:
            if self.latest_state == key:
                return strvalue
    state_display = property(_get_state_display)

    def store_dict_results(self, result_dict, debug=None):
        """ validate the dictionary structure results from the poller. If kosher, then use the returncode
        to set the current state for this monitor. Returns "True" if data passed validation
        and was stored.
        """
        validated = validate_return_dictionary(result_dict)
        if (validated):
            self.state = result_dict['returncode']
            self.latest_result_string = result_dict['decoded']['human']
            decoded = result_dict['decoded']
            keylist = decoded.keys()
            keylist.remove('human')
            for key in keylist:
                if (debug):
                    print "getting datastore named %s" % key
                (datastore, created_newds) = Datastore.objects.get_or_create(monitor=self, name=key)
                if (created_newds):
                    datastore.save()
                    datastore.create_rrd(overwrite=True)
                    if (debug):
                        print "!!! Created datastore for name %s" % key
                        print "!!! generated RRD file %s for key %s (id is %s)" % (datastore.location, key, datastore.id)
                insertvalue = decoded[key]['value']
                # convert timestamp from string to datetime object
                timestamp = dateutil.parser.parse(result_dict['timestamp'])
                if (debug):
                    print "Inserting %s into datastore %s for key %s (datastore name is %s)" % (insertvalue, datastore.location, key, datastore.name)
                datastore.insert_data(timestamp, insertvalue)
                datastore.save()
            self.save()
            return True
        return False

    def store_json_results(self, json_result_dict, debug=None):
        """ validate the JSON results from the poller. If kosher, then use the returncode
        to set the current state for this monitor. Returns "True" if data passed validation
        and was stored.
        """
        if (debug):
            print "!!! attempting to store data %s" % json_result_dict
        validated = validate_poller_results(json_result_dict)
        if (validated):
            if (debug):
                print "!!! data has been validated"
            result_dict = simplejson.loads(json_result_dict)
            return self.store_dict_results(result_dict, debug)

    class Meta:
        """META class for Django model administration - basic ordering, naming."""
        verbose_name, verbose_name_plural = "Monitor", "Monitors"

    def __unicode__(self):
        """unicode string representation of a Monitor"""
        return u"Monitor %s (%s) against %s" % (self.name, self.plugin_name, self.host)

    @models.permalink
    def get_absolute_url(self):
        """ returns the absolute URL for the monitor element"""
        return ('core.views.monitor_detail', [str(self.id)])

try:
    tagging.register(Monitor)
except tagging.AlreadyRegistered:
    # You might wonder what is going on here.
    # This is a bug in the svn release.  For some reason, the authors' cannot handle
    # re-registering gracefully, so we have to.
    # This should be fixed in later releases, but your code will still work here after it is.
    pass


class DatastoreManager(models.Manager):
    """ manager object for manipulating data store elements"""
    # def pending_update(self):
    #     """ returns the iterable django query set containing monitor objects that are due for updating"""
    #     basic_set = self.get_query_set().exclude(nextupdate__gte=datetime.datetime.now())
    #     return basic_set


class Datastore(models.Model):
    """ represents a set of data storage - backed by any number of different things.
    RRDtool data - direct into a database - etc
    Intended to represent a time-series set of numeric values (decimal, int, float, etc)
    """
    # pylint: disable=E1101
    objects = DatastoreManager()

    #GAUGE', 'COUNTER', 'DERIVE', 'ABSOLUTE', 'COMPUTE'
    DSTYPE_CHOICES = (
        ('GAUGE', u'Gauge'),
        ('COUNTER', u'Counter'),
        ('DERIVE', u'Derive'),
        ('ABSOLUTE', u'Absolute'),
        ('COMPUTE', u'Compute'),
    )

    monitor = models.ForeignKey(Monitor)
    name = models.CharField(max_length=18)  # name of data store - max 18 characters for RRD
    dstype = models.CharField(max_length=8, choices=DSTYPE_CHOICES, default='GAUGE')
    heartbeat = models.IntegerField(default=600)
    step = models.IntegerField(default=300)
    # TODO: IF ANY OF THESE VALUES ARE CHANGED, THE RRD FILE FOR THE DATASTORE ID SHOULD BE RECREATED
    # TODO: OTHERWISE THINGS WILL LIKELY BREAK ON STORAGE

    def _get_location(self):
        if self.id is None:
            return None
        else:
            first_level = "%d" % (self.id % 10)
            id_string = "%d.rrd" % self.id
            base_path = os.path.join(RRDFILE_ROOT, first_level)
            full_path = os.path.join(base_path, id_string)
            if not(os.path.exists(base_path)):
                os.mkdir(base_path)
            return full_path
    location = property(_get_location)

    def _rrd_exists(self):
        return os.path.exists(self.location)
    rrd_exists = property(_rrd_exists)

    def _make_rrd_file(self):
        dss = []
        rras = []
        ds1 = DS(dsName=self.name, dsType=self.dstype, heartbeat=self.heartbeat)
        dss.append(ds1)
        #86400 = 1 day, 604800 = 1 week, 2620800 = 1 month, 7862400 = 1 quarter
        # min, max, and average every 5 minutes for 3 months
        rra_5min_avg_qtr = RRA(cf='AVERAGE', xff=0.5, steps=1, rows=26208)
        rras.append(rra_5min_avg_qtr)
        rra_5min_min_qtr = RRA(cf='MIN', xff=0.5, steps=1, rows=26208)
        rras.append(rra_5min_min_qtr)
        rra_5min_max_qtr = RRA(cf='MAX', xff=0.5, steps=1, rows=26208)
        rras.append(rra_5min_max_qtr)
        # min, max, and average daily for 2 years
        rra_daily_avg_2yr = RRA(cf='AVERAGE', xff=0.5, steps=288, rows=730)
        rras.append(rra_daily_avg_2yr)
        rra_daily_min_2yr = RRA(cf='MIN', xff=0.5, steps=288, rows=730)
        rras.append(rra_daily_min_2yr)
        rra_daily_max_2yr = RRA(cf='MAX', xff=0.5, steps=288, rows=730)
        rras.append(rra_daily_max_2yr)
        # this takes up 633K per data source...
        just_a_bit_ago = epoch(datetime.datetime.now()) - 86400
        this_rrd_file = RRD(self.location, ds=dss, rra=rras, step=self.step, start=just_a_bit_ago)
        this_rrd_file.create()

    def _png_path(self):
        if self.id is None:
            return None
        else:
            first_level = "%d" % (self.id % 10)
            base_path = os.path.join(PNGFILE_ROOT, first_level)
            if not(os.path.exists(base_path)):
                os.mkdir(base_path)
            return base_path
    png_path = property(_png_path)

    def create_rrd(self, overwrite=False):
        """ creates an RRD file"""
        if os.path.exists(self.location):
            if (overwrite):
                self._make_rrd_file()
                return True
            else:
                return False
        else:
            self._make_rrd_file()
            return True

    def insert_data(self, timestamp, value):
        """
        responsible for inserting data (value) at the given timestamp (timestamp) into the RRD file associated
        with this datastore.

        This method will create an RRD file if it doesn't already exist
        """
        if not(self.rrd_exists):
            self.create_rrd()
            print "WARNING::: had to create RRD %s (ds is %s)" % (self.location, self.name, )
        this_rrd = RRD(self.location)
        # convert datetime object to seconds since epoch for RRD...
        epoch_int = epoch(timestamp)
        this_rrd.bufferValue(epoch_int, value)
        this_rrd.update()

    def generate_rrd_graph(self, duration=None, width=None, height=None, name_extension=None, location=None):
        """
        duration - # of seconds back to graph
        width - width of image generated
        height - height of image generated
        name_extension - graph name is id.png, name extension adds in 1-extension.png
        location = location for PNG file
        debug = enable print debugging

        TODO: do we want to enable color selection in through the method?
        """
        # set up objects to be displayed in graph...
        def1 = DEF(rrdfile=self.location, vname='example', dsName=self.name)
        vdef1 = VDEF(vname='myavg', rpn='%s,AVERAGE' % def1.vname)
        area1 = AREA(defObj=def1, color="#FFA902", legend=self.name)
        line1 = LINE(defObj=vdef1, color="#01FF13", legend='Average', stack=True)
        # set up graph details - colors, name, duration, size
        cattr = ColorAttributes()
        cattr.back = '#333333'
        cattr.canvas = '#333333'
        cattr.shadea = '#000000'
        cattr.shadeb = '#111111'
        cattr.mgrid = '#CCCCCC'
        cattr.axis = '#FFFFFF'
        cattr.frame = '#AAAAAA'
        cattr.font = '#FFFFFF'
        cattr.arrow = '#FFFFFF'

        graphname = '%s.png' % (self.id)
        if (name_extension):
            graphname = '%s-%s.png' % (self.id, name_extension)
        graphfile = graphname
        graphfile = os.path.join(self.png_path, graphname)
        if (location):
            graphfile = os.path.join(location, graphname)

        # Now that we've got everything set up, let's make a graph
        # default - 1 day
        now = datetime.datetime.now()
        endtime = epoch(now)
        day = 24 * 60 * 60
        starttime = endtime - day
        if (duration):
            starttime = endtime - duration
        # create the graph
        rrdgraph = Graph(graphfile, start=starttime, end=endtime, vertical_label=self.name, color=cattr)
        if (width):
            rrdgraph.width = width
        if (height):
            rrdgraph.height = height
        rrdgraph.data.extend([def1, vdef1, area1])
        rrdgraph.write()

    class Meta:
        """META class for Django model administration - basic ordering, naming."""
        ordering = ['name']

    def __unicode__(self):
        """unicode string representation of a Datastore"""
        return u"Datastore %s for %s" % (self.name, self.monitor)

    @models.permalink
    def get_absolute_url(self):
        """ returns the absolute URL for the monitor element"""
        return ('core.views.datastore_detail', [str(self.id)])