notomorrow.de code:notes

code:notes:icinga

anonymous@undisclosed.example.com (Anonymous) — Wed, 03 Sep 2014 16:32:09 +0000

Icinga Install Process

following on centos http://docs.icinga.org/1.0/en/quickstart-icinga.html see icinga_rules for a yet unfinished paste of the configuration.

## some debian deps
sudo aptitude install libdbi-dev fcgiwrap

cd
mkdir src
cd src
git clone git://git.icinga.org/icinga-core.git
cd icinga-core/
./configure
make
make all
sudo useradd -r icinga
sudo make fullinstall
sudo make install-config
sudo make install-webconf
sudo vim /usr/local/icinga/etc/objects/contacts.cfg
sudo htpasswd -c /usr/local/icinga/etc/htpasswd.users icingaadmin
sudo service httpd restart
sudo chkconfig httpd on
cd ~/src/
wget https://nagios-plugins.org/download/snapshot/nagios-plugins-master.tar.gz
tar tfvz nagios-plugins-master.tar.gz
tar xfvz nagios-plugins-master.tar.gz
rm nagios-plugins-master.tar.gz
cd nagios-plugins-*/
./configure --prefix=/usr/local/icinga --with-nagios-user=icinga
make
sudo make install

nginx config

## Icinga Classic - Configuration                                                                                                                  location /icinga {                                                                                                                                   alias   /usr/local/icinga/share;
      index  index.html;
      auth_basic              "Icinga Access";
      auth_basic_user_file    /usr/local/icinga/etc/htpasswd.users;
  }
 
  location ~ /icinga/(.*)\.cgi$ {
      root /usr/local/icinga/sbin;
      rewrite ^/icinga/cgi-bin/(.*)\.cgi /$1.cgi break;
      include /etc/nginx/fastcgi_params;
      fastcgi_pass  unix:/var/run/fcgiwrap.socket;
      fastcgi_index index.php;
      fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
      auth_basic              "Icinga Access";
      auth_basic_user_file    /usr/local/icinga/etc/htpasswd.users;
      fastcgi_param  AUTH_USER          $remote_user;
      fastcgi_param  REMOTE_USER        $remote_user;
  }

distributed services

uses active and passive checks, rest is timing.

     
define service{
 #use                     distributed-service   ; template to inherit from
 use generic-service
 name                    distributed-service   ; name of this template
 ;active_checks_enabled   0                 ; no active checks
 check_command               service-is-stale
 passive_checks_enabled  1                 ; allow passive checks
 check_period            24x7              ; check active all the time
 check_freshness         1
 register                0                 ; this is a template, not a real service
 }

code:notes:ipmi

anonymous@undisclosed.example.com (Anonymous) — Wed, 06 Sep 2017 21:02:51 +0000

== ipmitool

ipmitool -I lanplus -H $host -U adm sol activate

code:notes:nginx

anonymous@undisclosed.example.com (Anonymous) — Fri, 15 Aug 2014 02:12:20 +0000

NGINX

SSL

      ssl on;
      ssl_dhparam /etc/nginx/keys/dh4096.pem;
      ssl_protocols TLSv1.2 TLSv1 SSLv3;
      ssl_ciphers ALL:!aNULL:!eNULL:!ADH:!EXP:!DES:!MEDIUM:!LOW:kEDH+DHE:SHA256:RC4+RSA;
      ssl_prefer_server_ciphers   on;
      ssl_session_timeout  5m;
      ssl_session_cache    shared:SSL:10m;
      ssl_certificate  /etc/nginx/keys/ntmr.crt;
      ssl_certificate_key /etc/nginx/keys/ntmr.key;

Proxy

strip ssl and adjust server vars for cgi scripts

  
location / {
  proxy_pass  http://www;                                                                         
  proxy_set_header        Accept-Encoding   "";
  proxy_set_header        Host            $host;
  proxy_set_header        scheme          $scheme;
  proxy_set_header        X-Real-IP       $remote_addr;
  proxy_set_header        X-Forwarded-Port       $server_port; 
  proxy_set_header        X-Forwarded-For        $proxy_add_x_forwarded_for;
  proxy_set_header        X-Forwarded-Proto      $scheme;
        
  proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
  
  proxy_redirect off;
}

ww1

global scope

map $http_x_forwarded_port $external_port {
  default $http_x_forwarded_port;
  '' $server_port;
}

cgi config

include fastcgi_params;
fastcgi_param  SERVER_PORT          $external_port;

code:notes:osm_roads

anonymous@undisclosed.example.com (Anonymous) — Fri, 15 Aug 2014 02:10:31 +0000

OpenStreetMap Road Data

download street to postcode relations from geofabrik (check Bandwidth and Connection Rate Limits) and save to mysql

sql:

create table osm_roads( id integer, name char( 255 ), category char( 255 ), postal_code char( 255 ));

shell:

wget -e robots=off -nd -r -l1 -np -A *.bz2 http://download.geofabrik.de/openstreetmap/europe/germany/
wget -O - https://notomorrow.de/_export/code/notess/osm_roads?codeblock=0 > osm_roads.py
wget -O - https://notomorrow.de/_export/code/notes/osm_roads?codeblock=1 > osm_roads.sql
mysql < osm_roads.sql
for i in *.bz2; do bunzip2 $i && python osm_roads.py ${i%.bz2} && rm ${i%.bz2}; done;

todo: save cleaned street values, like ccity for cities

osm_roads.py

import sys
import MySQLdb
 
from pprint import pprint
#pp = pprint.PrettyPrinter(indent=4)
 
idx_replace = [ ' ', '-', '.', ',', '(', ')' ]
def clean( s ):
    s = s.lower( )
    for c in idx_replace: s = s.replace( c, '' )
    return s
 
def indexstr( l ): return '|'.join( map( clean, l ))
 
def save( act, data ):
 
    #pprint( { 'save': data })
 
 
    if 'addr:city' in data:     data['city']     = data['addr:city']
    if 'addr:country' in data:  data['country']  = data['addr:country']
    if 'addr:postcode' in data: data['postcode'] = data['addr:postcode']
    if 'addr:street' in data:   data['street']   = data['addr:street']
 
    if 'highway' in data:
        data['category'] = data['highway']
        if not 'steet' in data and 'name' in data:
            data['street'] = data['name']
 
    if not 'postcode' in data and 'postal_code' in data:        data['postcode'] = data['postal_code']
    if not 'category' in data and 'building' in data:   data['category'] = 'building'
 
    if not 'street' in data: return False
    if not 'postcode' in data or len( data['postcode'] ) != 5: return False
 
    if not 'category' in data:  data['category'] = 'addr'
    if not 'city' in data:      data['city'] = ''
    data['way_id'] = data['way_id']
 
    found = { }
    idx = ''
 
    ## postcode from geo_db
    if data['city'] == '':
        c1.execute ( 'select name, sortname from geo_place where postcode=%s',( data['postcode'] ))
        found = c1.fetchone( )
        if found is not None:
            pprint({ 'city found': found })
            data['city'] = found[0]
            data['ccity'] = found[1]
 
    ## check existing
    c1.execute ( 'select id, city from osm_roads3 where street=%s and category=%s and postcode=%s and city=%s',
        ( data['street'], data['category'], data['postcode'], data['city'] ))
    found = c1.fetchone( )
    if found is not None:
        pprint({ 'road found': found })
        return True;
 
    ## insert roads
    if not 'ccity' in data: data['ccity'] = clean( data['city'] )
    elif data['ccity'] != '': data['ccity'] = clean( data['ccity'] )
 
    c2.execute ("""insert into osm_roads3( way_id, street, category, postcode, city, ccity )
                                VALUES( %s, %s, %s, %s, %s, %s )""",
        ( data['way_id'], data['street'], data['category'], data['postcode'], data['city'], data['ccity'] ))
    print "ins: %s" % data['way_id']
 
   if data['city'] != '':
        ## update plz
        c1.execute ( 'select id from osm_plz where postcode=%s and city=%s',
            ( data['postcode'], data['city'] ))
        if c1.fetchone( ) is None:
            print "plz: %s - %s" % ( data['postcode'], data['city'] )
            c2.execute ("""insert into osm_plz( postcode, city ) VALUES( %s, %s )""",
            ( data['postcode'], data['city'] ))
 
        ## update index
        idx = indexstr([ data['postcode'], data['city'], data['street'] ])
        print "idx: %s" % idx
        c2.execute ("""insert into osm_idx( id, idx ) VALUES( %s, %s )""",
                ( data['way_id'], idx ))
 
from lxml import etree
def read_dump( infile ):
    data = {}
    f = open( infile ,'r' )
    for ev, el in etree.iterparse( infile, events=( 'start', 'end' ) ):
        tag = el.tag
        if ev == 'start':
            if tag == 'way':
                k = data['way_id'] = el.attrib.get( 'id' );
            elif 'way_id' in data:
                if tag == 'tag':
                    a = el.attrib
                    data[a.get( 'k' )] = a.get( 'v' ).encode( 'utf8' )
        else:
            if tag == 'way':
                save( tag, data );
                data = {}
        el.clear( )
 
try:
    conn = MySQLdb.connect (host = "localhost",user = "xxx",passwd = "xxx",db = "xxx")
    c1 = conn.cursor( )
    c2 = conn.cursor ()
except MySQLdb.Error, e:
    print "Error %d: %s" % (e.args[0], e.args[1])
    sys.exit (1)
 
print "readfile %s" % sys.argv[1]
read_dump( sys.argv[1] )
print "commit"
conn.commit( )
c1.close( )
c2.close( )
conn.close( )
sys.exit( )

osm_roads3.sql

CREATE DATABASE roads CHARACTER SET utf8;
CONNECT roads;
CREATE TABLE `osm_roads3` (
  `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
  `way_id` CHAR(32) DEFAULT NULL,
  `category` CHAR(32) DEFAULT NULL,
  `postcode` CHAR(5) DEFAULT NULL,
  `street` CHAR(124) DEFAULT NULL,
  `city` CHAR(124) DEFAULT NULL,
  `idx` CHAR(255) DEFAULT NULL,
  `ccity` CHAR(255) DEFAULT NULL,
  UNIQUE KEY `id` (`id`)
);
CREATE TABLE `osm_idx` (
  `id` BIGINT(20) DEFAULT NULL,
  `idx` CHAR(255) DEFAULT NULL
);
CREATE TABLE `osm_plz` (
  `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
  `postcode` CHAR(5) DEFAULT NULL,
  `city` CHAR(255) DEFAULT NULL,
  UNIQUE KEY `id` (`id`)
);
-- prefill with postcodes
CREATE TABLE `geo_place` (
  `id` INT(11) DEFAULT NULL,
  `postcode` CHAR(255) DEFAULT NULL,
  `category` CHAR(255) DEFAULT NULL,
  `name` CHAR(255) DEFAULT NULL,
  `sortname` CHAR(255) DEFAULT NULL
);

code:notes:php-fpm

anonymous@undisclosed.example.com (Anonymous) — Fri, 15 Aug 2014 02:11:15 +0000

php

add php fpm service as user ntmr

adduser –system –home /devnull –no-create-home –group –disabled-login –gecos “ntmr” ntmr

fpm config

/etc/php5/fpm/pool.d/ntmr.conf

[notomorrow.de]
 
user = ntmr
group = ntmr
listen = '/tmp/php-fpm-ntmr.sock'
pm = dynamic
pm.max_children = 5
pm.start_servers = 2
pm.min_spare_servers = 1
pm.max_spare_servers = 3

nginx config

/etc/nginx/inc/ntmr.php.inc

location ~ \.php$ {
  fastcgi_split_path_info ^(.+\.php)(/.+)$;
  include        fastcgi_params;
 
  fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
 
  fastcgi_intercept_errors on;
  fastcgi_pass unix:/tmp/php-fpm-ntmr.sock;
}

sample host

        root   /srv/ntmr;
        index  index.php index.html;
 
         location / {
           try_files $uri $uri/index.php; }
 
        include        conf.d/ntmr.php.inc;
 
        ssl on;
        ssl_certificate  /etc/nginx/keys/mail.ntmr.crt;
        ssl_certificate_key /etc/nginx/keys/mail.ntmr.key;                                                                                    
 
        ssl_session_timeout  5m;
        ssl_protocols  SSLv2  TLSv1;
        ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;
        ssl_prefer_server_ciphers   on;

php-apps

dokuwiki

code:notes:postgres

anonymous@undisclosed.example.com (Anonymous) — Fri, 15 Aug 2014 02:13:53 +0000

== Postgres replication install

=== debian

requirements

apt-get install wget get build-essential libkrb5-dev
apt-get install libxslt-dev libxml2-dev libpam-dev libedit-dev

wget -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | apt-key add -
apt-get install postgresql-9.1
echo "deb http://apt.postgresql.org/pub/repos/apt/ squeeze-pgdg main" \
  >> /etc/apt/sources.list.d/pgdg.list

(cat <<-DEMO
Package: *
Pin: release o=apt.postgresql.org
Pin-Priority: 500
DEMO
) >> /etc/apt/preferences.d/pgdg.pref

apt-get update
apt-get install pgdg-keyring
apt-get install postgresql-9.2 postgresql-server-dev-9.2

repmgr

git clone git://github.com/2ndQuadrant/repmgr.git
cd repmgr
make clean
make USE_PGXS=1
make USE_PGXS=1 install

update-alternatives --install /usr/bin/repmgr repmgr /usr/lib/postgresql/9.2/bin/repmgr 10
update-alternatives --install /usr/bin/repmgr repmgr /usr/lib/postgresql/9.2/bin/repmgrd 10

=== centos

yum install repmgr

repmgr               x86_64               1.2.0-1.rhel6                 pgdg91                59 k

notomorrow.de code:notes

code:notes:icinga

Icinga Install Process

nginx config

Links

distributed services

code:notes:ipmi

code:notes:nginx

NGINX

SSL

Proxy

ww1

code:notes:osm_roads

OpenStreetMap Road Data

code:notes:php-fpm

php

fpm config

nginx config

sample host

php-apps

code:notes:postgres