01 September 2011

Creating new Vim filetype for server logs

In this post I'll show how to create a simple Vim syntax script to highlight source of common error and access logs.

First make sure you have the following folders and files:
~/.vimrc		- the main config
~/.vim/ftdetect		- directory with filetype scripts
~/.vim/syntax		- directory with specific syntax scripts
If something missing, you can copy it from $VIMRUNTIME directory. Find block starting with
if has("autocmd")
and add the following line there:
au BufRead,BufNewFile *.log set filetype=error_log

Create filetype script in ~/.vim/ftdetect/error_log.vim:

if did_filetype()	" filetype already set..
	finish		" ..don't do these checks
endif

au BufRead,BufNewFile *.error.log set filetype=error_log au BufRead,BufNewFile *.access.log set filetype=error_log
Finally you need syntax script itself, ~/.vim/syntax/error_log.vim:
" Vim syntax file
" Language:    error_log
" Maintainer:  Ruslan Osmanov <rrosmanov at gmail dot com>
" Last Change: Thu Sep  1 15:30:52 MSD 2011
" Version:     1.0

if version < 600
  syntax clear
elseif exists("b:current_syntax")
  finish
endif

" Always ignore case
syn case ignore

" General keywords which don't fall into other categories
syn keyword error_logKeyword 			error warning notice 

" Special values
syn keyword error_logSpecial         	referer client

" Strings (single- and double-quote)
syn region error_logString           start=+"+  skip=+\\\\\|\\"+  end=+"+
syn region error_logString           start=+'+  skip=+\\\\\|\\'+  end=+'+

" Numbers and hexidecimal values
syn match error_logNumber            "-\=\<[0-9]*\>"
syn match error_logNumber            "-\=\<[0-9]*\.[0-9]*\>"
syn match error_logNumber            "-\=\<[0-9][0-9]*e[+-]\=[0-9]*\>"
syn match error_logNumber            "-\=\<[0-9]*\.[0-9]*e[+-]\=[0-9]*\>"
syn match error_logNumber            "\<0x[abcdefABCDEF0-9]*\>"

" IP addresses
syn match error_logIP            	"-\=\<[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*\>"

syn region error_logOperator         start="referrer\:" end="" contains=ALL

" URI
syn match error_logURI            	"http\:\/\/[^ ]*"

" Date 
syn match error_logDate 			"\[[0-9]*\/[a-zA-Z]\{3\}\/\d\{4\}:\d\{2\}:\d\{2\}:\d\{2\} +\d\{4\}\]"
syn match error_logDate				"\[[a-zA-Z]\{3\} [a-zA-Z]\{3\} \d\{2\} \d\{2\}:\d\{2\}:\d\{2\} \d\{4\}\]"

" Domain
syn match error_logDomain            "\(www\.\)\?[a-z0-9\-\_]*\.\(ru\|com\|by\|uz\|ua\|kz\|su\|net\|org\)"

" Define the default highlighting.
" For version 5.7 and earlier: only when not done already
" For version 5.8 and later: only when an item doesn't have highlighting yet
if version >= 508 || !exists("did_error_log_syn_inits")
	if version < 508
		let did_error_log_syn_inits = 1
		command -nargs=+ HiLink hi link <args>
	else
		command -nargs=+ HiLink hi def link <args>
	endif

	HiLink error_logKeyword         Statement
	HiLink error_logIP				Type
	HiLink error_logSpecial  	    Special
	HiLink error_logString          String
	HiLink error_logNumber          Number
	HiLink error_logDate			Comment 
	HiLink error_logURI				Underlined 
	HiLink error_logDomain			Underlined 

	delcommand HiLink
endif

let b:current_syntax = "error_log"
" vim:sw=4:

03 August 2011

Colorized svn status

I post some shell scripts here to output svn status command output colorized.

Bash Script

#!/bin/bash - 

set -o nounset                              # Treat unset variables as an error

txtblk='\e[0;30m' # Black - Regular
txtred='\e[0;31m' # Red
txtgrn='\e[0;32m' # Green
txtylw='\e[0;33m' # Yellow
txtblu='\e[0;34m' # Blue
txtpur='\e[0;35m' # Purple
txtcyn='\e[0;36m' # Cyan
txtwht='\e[0;37m' # White
bldblk='\e[1;30m' # Black - Bold
bldred='\e[1;31m' # Red
bldgrn='\e[1;32m' # Green
bldylw='\e[1;33m' # Yellow
bldblu='\e[1;34m' # Blue
bldpur='\e[1;35m' # Purple
bldcyn='\e[1;36m' # Cyan
bldwht='\e[1;37m' # White
unkblk='\e[4;30m' # Black - Underline
undred='\e[4;31m' # Red
undgrn='\e[4;32m' # Green
undylw='\e[4;33m' # Yellow
undblu='\e[4;34m' # Blue
undpur='\e[4;35m' # Purple
undcyn='\e[4;36m' # Cyan
undwht='\e[4;37m' # White
bakblk='\e[40m'   # Black - Background
bakred='\e[41m'   # Red
badgrn='\e[42m'   # Green
bakylw='\e[43m'   # Yellow
bakblu='\e[44m'   # Blue
bakpur='\e[45m'   # Purple
bakcyn='\e[46m'   # Cyan
bakwht='\e[47m'   # White
txtrst='\e[0m'    # Text Reset

RESULT="`svn st`"
echo "$RESULT" | while read LINE
do
 case "${LINE:0:1}" in
  'M')
   echo -e "$txtylw$LINE$txtrst"
   ;;
  'X')
   echo -e "$bakpur$LINE$txtrst"
   ;;
  '?')
   echo -e "$txtgrn$LINE$txtrst"
   ;;
  'D')
   echo -e "$bakylw$LINE$txtrst"
   ;;
  'I')
   echo -e "$txtblu$LINE$txtrst"
   ;;
  *)
   echo -e "$LINE";
   ;;
 esac
done

AWK Script

#!/usr/bin/awk -f
BEGIN {
  cmd = "svn st";
  while (cmd | getline) {
    char = substr($1, 0, 1);
    if (char == "M") {
      print "\033[36m(", $1, ")    ", $2, $3, "\033[0m"; 
    } else if (char == "X") {  
    print "\033[35m(", $1, ")    ", $2, "\033[0m"; 
    } else if (char  == "A") {  
      print "\033[32m(", $1, ")    ",  $2, $3, "\033[0m"; 
    } else if (char == "?") { 
      print "\033[1;31m(", $1, ")    ", $2, $3, "\033[0m"; 
    } else if (length($1) == 1 ) {
      print $1,"      " $2; 
    } else if ($1 == "---") { # changelists etc.
      print "\033[0;40m", $1, $2, $3, $4, "\033[0m"; 
    } else {
      print $0;
    }
  }
}

29 July 2011

Mapping special characters in Sphinx configuration

Sphinx sometimes assumes some characters as word separators. For instance, letters 'e' and 'ё' are kinda similar in Russian. Some print issues even replace the latter with the former. However, Sphinx assumes 'ё' a word separator. To prevent it, one should search for Unicode code points and append mapping to charset_table index config:
charset_type  = utf-8
charset_table  = 0..9, A..Z->a..z, _, a..z, U+A8->U+E5, U+B8->U+E5, U+410..U+42F->U+430..U+44F, U+430..U+44F, \
U+0451->U+0435
Here U+0451->U+0435 maps 'ё'(U+0451) to 'e'(U+0435). The Unicode code points could be found here.

References

16 June 2011

How to create a Sphinx wordform dictionary

To build a wordform file for Sphinx you may use some kind of spellchecker dictionary like myspell, ispell, pspell, aspell. Let's make a wordform file for Russian language from myspell-russian package in openSUSE.

To install myspell-russian:
$ sudo zypper in myspell-russian
$ rpm -ql myspell-russian
/usr/share/doc/packages/myspell-russian
/usr/share/doc/packages/myspell-russian/descr_en.txt
/usr/share/doc/packages/myspell-russian/descr_ru.txt
/usr/share/doc/packages/myspell-russian/description.xml
/usr/share/doc/packages/myspell-russian/dictionaries.xcu
/usr/share/doc/packages/myspell-russian/icon.png
/usr/share/doc/packages/myspell-russian/licence.txt
/usr/share/myspell
/usr/share/myspell/ru_RU.aff
/usr/share/myspell/ru_RU.dic

Build wordforms:
$ spelldump /usr/share/myspell/ru_RU.dic /usr/share/myspell/ru_RU.aff wordforms_myspell_ru_RU.txt
spelldump, an ispell dictionary dumper

Loading dictionary...
Loading affix file...
Using MySpell affix file format
Dictionary words processed: 146265

Detect encoding:
$ cat wordforms_myspell_ru_RU.txt | enca -L ru 
KOI8-R Cyrillic
  LF line terminators

Конвертируем в UTF-8:
$ iconv -f KOI8-R -t UTF-8 -o wordforms_myspell_ru_RU_UTF8.txt wordforms_myspell_ru_RU.txt

Finally, in sphinx.conf set:
wordforms     = /path/to/wordforms_ru_RU_UTF8.txt

See also

23 April 2011

Strange C sizeof(struct) results?

If you wonder, why sizeof operator returns values bigger than you expect, you probably have to learn about compiler data structure alignment(as I had to:). Here I post a simple program demonstrating the case.

main.c

#include <stdio.h>
#include <string.h>

typedef struct 
#ifdef PACKED 
__attribute__ ((packed)) 
#endif
{
    char *str;      // 8
    char ch ;       // 1
    int a;          // 4
    int b;          // 4
    int c;          // 4
    // Packed total size should be: 21
    // Aligned total size should be: 24
} test_t;

static void print_sizes(test_t *t)
{
    printf("DATA TYPE SIZES\n");
    printf("char*: %ld\n", sizeof(char*));
    printf("char: %ld\n", sizeof(char));
    printf("int: %ld\n", sizeof(int));

    printf("\nSIZE OF *t: %ld\n", sizeof(*t));
}

int main (int argc, char const* argv[]) {
    test_t t;
    t.str = "test string"; 

    print_sizes(&t);

    return 0;
}

makefile

ifeq ($(PACKED), true)
    PACK=-DPACKED
else
    PACK=
endif

ifdef OUT
    OUTFILE=$(OUT)
else
    OUTFILE=test
endif

CC=gcc
CFLAGS=-Wall -c $(PACK)

all: test

test: main.o
    $(CC) main.o -o $(OUT) 

main.o: main.c
    $(CC) $(CFLAGS) main.c -o main.o

clean: 
    rm -f *.o *~

Compile versions with packed and padded memory:

$ make PACKED=true OUT=test_packed
$ make OUT=test_padded

and test them:
$ ./test_packed
DATA TYPE SIZES
char*: 8
char: 1
int: 4

SIZE OF *t: 21

$ ./test_padded
DATA TYPE SIZES
char*: 8
char: 1
int: 4

SIZE OF *t: 24

So one should consider a space-time tradeoff for every particular program.

I'd be glad, if it helps someone.

19 April 2011

How to add new highlight mode in GEdit

Here I show common steps to create custom language for GtkSourceView 2.0(GEdit uses it to display code). Then we'll create MBox language and special highlighting for it.

Common steps

1. Make directories for language definition and for new MIME:

$ mkdir -p ~/.local/share/gtksourceview-2.0/language-specs/
$ mkdir -p  ~/.local/share/mime/packages/

2. Create new MIME definition in ~/.local/share/mime/packages/

3. Create .lang file in ~/.local/share/gtksourceview-2.0/language-specs/

4. Update MIME database

$ update-mime-database ~/.local/share/mime

New language for MBox

STEP - 2: New MIME
Actually it should be already in /usr/share/mime/application/mbox.xml. But one would probably override it:

$ cat > ~/.local/share/mime/packages/mbox.xml
<!--
Author:     Ruslan Osmanov
Date:       Tue Apr 19 05:12:58 UTC 2011
Version:    1.0
-->
<mime-info xmlns='http://www.freedesktop.org/standards/shared-mime-info'>
    <mime-type type="application/mbox">
        <comment>MBox Source</comment>
        <magic priority="50">
            <match type="string" offset="0" value="&gt;"/>
        </magic>
        <glob pattern="*.mbox"/>
    </mime-type>
</mime-info>
^D
^D stands for Ctrl + D shortcut.

STEP - 3: New Language Definition

The GNOME Language Definition Reference 2.0[1] and Language Definition v2.0 Tutorial [2] are good start points.
Syntax highlighting styles could be found in /usr/share/gtksourceview-2.0/styles/.

This info is quite enought to implement a new language:

$ cat > ~/.local/share/gtksourceview-2.0/language-specs/mbox.lang
<?xml version="1.0" encoding="UTF-8"?>
<!--
Author: Ruslan Osmanov 
Date:   Mon Apr 18 17:08:52 UTC 2011
Version:   1.0
MBox language spec
-->
<language id="mbox" _name="MBox" version="2.0" _section="Email">
  <metadata>
 <property name="mimetypes">text/x-mbox;application/mbox</property>
 <property name="globs">*.mbox</property>
 <property name="line-comment-start">&gt;</property>
  </metadata>
  <styles>
 <style id="keyword" _name="Keyword" map-to="def:keyword"/>
 <style id="underlined" _name="Underlined" map-to="def:underlined"/>
 <style id="comment" _name="Comment" map-to="def:comment"/>
 <style id="string" _name="A string" map-to="def:string"/>
  </styles>
  <definitions>
 <context id="mbox">
   <include>
  <context id="keyword" style-ref="keyword">
    <prefix>^</prefix>
    <suffix>( |\:)</suffix>
    <keyword>(From|To|Subject)</keyword>
    <!-- Russian -->
    <keyword>(От|Кому|Тема)</keyword>
  </context>
  <context id="link" style-ref="underlined">
   <keyword>\b(ftp|http|https)\:\/\/.*\b</keyword> 
   <keyword>mailto\:.+\@[a-z]+</keyword> 
  </context>
  <context id="comment" style-ref="comment">
    <start>^&gt;</start>
    <end>$</end>
    <include>
   <context ref="link"/>
    </include>
  </context>
  <context id="comment-multiline" style-ref="comment">
    <start>^--</start>
    <include>
   <context ref="def:in-comment"/>
    </include>
  </context>
  <context id="string" end-at-line-end="true" style-ref="string">
    <start>"</start>
    <end>"</end>
  </context>

   </include>
 </context>
  </definitions>
</language>
<!--vim: set ft=xml:-->

^D
Note, ^D stands for Ctrl + D shortcut.

STEP - 4. Update MIME database

$ update-mime-database ~/.local/share/mime

Voila! You now have new MBox language in View - Highlight Mode - Email section.

References

08 April 2011

Change indentation: spaces to tabs

To convert space indentation in significant number of files, you may use the
following script.

#!/bin/bash - 
#===============================================================================
#          FILE:  fixtab.sh
#         USAGE:  ./fixtab.sh <directory>
#   DESCRIPTION:  Convert expanded tabs into tabs with 4 space width
#       OPTIONS: $1 directory
#        AUTHOR: Ruslan Osmanov 
#       CREATED: 04/08/2011 03:26:00 PM UZT
#===============================================================================
set -o nounset                              # Treat unset variables as an error

DIR=$1

function usage(){
 echo "bash $0 <directory>"
}

[[ ! -d $DIR ]] && usage

# Correct existing modelines I used to include modelines beginning from set
# expandtab, so this worked for me. If you don't use modelines, comment it out 
find $DIR -name *.php -exec \
 sed -i -e 's%# vim: set expandtab:%# vim: set noet:%g' {} \;

# Retab & format source code
find $DIR -name *.php -exec \
 vim -u NONE \
 -c 'set ft=php' \
 -c 'set shiftwidth=4' \
 -c 'set tabstop=4' \
 -c 'set noexpandtab!' \
 -c 'set noet' \
 -c 'retab!' \
 -c 'bufdo! "execute normal gg=G"' \
 -c wq \
 {} \;

For example:
$ bash ./fixtab.sh /srv/www/myproject
I would be glad, if this helped somebody.