Thursday, May 18, 2017

Send Large attachments with google API for gmail

The problem: How to send large size images using google's gmail API for python. The documentation is not very good and a 10MB size limit is not mentioned. Following their quickstart example, I use body-media instead of body to send large files.
See below for an example:
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import logging
log = logging.getLogger("log")
import httplib2from StringIO import StringIO
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/gmail-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'# mail size limit. can get up to 35MB
MAILSIZELIMIT = 1024*1024*25 # set to 25MB
def get_credentials():
    """Gets valid user credentials from storage.

    If nothing has been stored, or if the stored credentials are invalid,
    the OAuth2 flow is completed to obtain the new credentials.

    Returns:
        Credentials, the obtained credential.
    """
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'gmail-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_known_args()
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        log.debug('Storing credentials to ' + credential_path)
    return credentials

def jpg2attachment(filename):
  """Creates an image message from file on disk.
       See other examples here: https://developers.google.com/gmail/api/guides/sending"""
  with open(filename,'rb') as f:
    img = MIMEImage(f.read())
    img.add_header('Content-Disposition', 'attachment', filename=os.path.split(filename)[1])
  return img

def send_gmail_with_attachments(sender,recipients,subject,txtcontent,attachmentfilelist=[]):
  """Sends email with attachments.
       sender: string of sender name/address
       recipients: a list of strings with recipients emails
       subject: string of the subject
       txtcontent: string with textual message body
       attachmentfilelist: a list of file names. in this example it is only for jpg files.
  """
  log.info('Sending email to: {}'.format(recipients))
  log.debug('Connecting to gmail server')
  try:
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
  except:
    log.error('Error with credentials')
    return False
  for i in range(3): # 3 attempts to connect to service.
    try:
      service = discovery.build('gmail', 'v1', http=http)
      break
    except httplib2.ServerNotFoundError,e:
      if i==2:
        log.error('{} of 3 attempts to connect to mail server failed.'.format(i+1))
        log.error(e)
        return False  
  COMMASPACE = ', '
  toaddrs = COMMASPACE.join(recipients) # get recipients list
  # Create the container (outer) email message.
  msg = MIMEMultipart()
  msg['Subject'] = subject
  msg['From'] = sender
  msg['To'] = toaddrs
  # add content
  txt = MIMEText('text', 'plain')
  txt.set_payload(txtcontent)
  msg.attach(txt)
  raw = msg.as_string()
  attachments = [jpg2attachment(a) for a in attachmentfilelist if a.endswith('jpg')]
  # send the message via gmail
  attempt = 1
  while len(attachments): # as long as we have attachements
    mediafile = StringIO() # placeholder for media body file
    while len(attachments) and len(raw)+len(attachments[0].as_string())<MAILSIZELIMIT: # make sure we are not exceeding size limit.
      msg.attach(attachments.pop(0)) # remove image attachment from list and put in message
      raw = msg.as_string() # serialize message
    mediafile.write(raw) # populate the media body file
    media = MediaIoBaseUpload(mediafile,mimetype='message/rfc822',chunksize=1024*1024,resumable=True) # pack as a media message
    if len(msg.get_payload())>1: # make sure message is not empty (in case a single attachment is over size limit
      try:  
        service.users().messages().send(userId='me',body={},media_body=media).execute() # send the message
      except Exception,e:
        log.error("attampt {}. Can't send email.\n\n{}\n\n".format(attempt,e))
        if attempt<=3:
          attempt+=1
        else:
          figures = [img.get_filename() for img in msg.get_payload() if img.get_content_maintype()=='image']+[img.get_filename() for img in attachments]
          log.error('Failed to send figures: {}'.format(figures))
          return
      else:
        sentimages = [img for img in msg.get_payload() if img.get_content_maintype()=='image']
        log.info('Sent figures: {}'.format([img.get_filename() for img in sentimages]))
        [msg.get_payload().remove(img) for img in sentimages]# remove figures from message
    elif len(attachments): # there is a too large image, send to last or remove.
      img = attachments.pop(0)
      if not len(img.as_string())>MAILSIZELIMIT:
        log.debug('Large size attachment {} ({}) pushed to end of line.'.format(img.get_filename(),len(img.as_string())))
        attachments.append(img) # push to end of line
      else:
        log.error('Failed to send figure: {}, size: {}'.format(img.get_filename(),len((img.as_string()))))
  return True

Wednesday, January 6, 2016

Real-Time Kernel for RaspberryPi 2

This post will describe how to compile a RT-kernel for RaspberryPi 2.
I'm using UBUNTU 14.04 LTS 64bit for compiling Raspbian Jessie.
After a long time, I needed a RT kernel for a new project with the new RaspberryPi2. Realizing My old post has broken links, I struggled again with the task. This time, after a successful attempt, I'm determined to put it all here step by step.

##### Tested on UBUNTU 14.04 LTS with Rasbian Jessie 4.1
# install needed packages for compiling the kernel
sudo apt-get install libncurses5-dev libncursesw5-dev
#Download Raspberry Pi tools:
git clone https://github.com/raspberrypi/tools.git
# Download source files and patches
git clone -b rpi-4.1.y --depth 1 git://github.com/raspberrypi/linux.git linux-rpi-4.1.y
# Download RT patch 
wget https://www.kernel.org/pub/linux/kernel/projects/rt/4.1/patch-4.1.15-rt17.patch.xz
xz -d patch-4.1.15-rt17.patch.xz
# patch the kernel. This step might produce errors
# if rt-patch and the kernel don't match
cd linux-rpi-4.1.y
cat ../patch-4.1.15-rt17.patch | patch -p1
#Export the following variables to specify cross-compilation options:
export ARCH=arm 
#to compile on a x64-machine you need to export:
export CROSS_COMPILE=~/tools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64/bin/arm-linux-gnueabihf-
#create a config file for Raspberry Pi 2:
make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm bcm2709_defconfig
# make changes to the config
make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm menuconfig
enable Preemption model -> Fully preemptible Kernel (RT)
change timer freq to 1000Hz
enable device tree (all options)
#Compile the kernel:
make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm -j5
#Install modules, will result in "lib" folder with modules and firmware:
mkdir kernel-rt
INSTALL_MOD_PATH=kernel-rt make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm modules_install
## Install the kernel to non-RT image:
Copy arch/arm/boot/Image to /boot/kernel7.img on SD card.
Copy arch/arm/boot/dts/bcm2709-rpi-2-b.dtb to /boot/bcm2709-rpi-2-b.dtb on SD card
# add to /boot/config.txt
echo “device_tree=bcm2709-rpi-2-b.dtb” >> /boot/config.txt
Copy (merge if necessary) kernel-rt/lib to / on SD card.

## Testing RT-Image
Insert SD card and connect RPI to power, keyboard and screen. 
Login with user pi and password raspberry
#Installing cyclictest utility on Raspberry Pi:
git clone http://git.kernel.org/pub/scm/linux/kernel/git/clrkwllms/rt-tests.git 
cd rt-tests
make all
cp ./cyclictest /usr/bin/
cd ~
sudo cyclictest -l100000 -m -n -a0 -t1 -p99 -i400 -h400 -q

# WITH NO RT RESULT:
# Total: 000100000
# Min Latencies: 00015
# Avg Latencies: 00021
# Max Latencies: 00118
# Histogram Overflows: 00000
# Histogram Overflow at cycle number:
# Thread 0:

# WITH RT RESULT:
# Total: 000100000
# Min Latencies: 00013
# Avg Latencies: 00020
# Max Latencies: 00066
# Histogram Overflows: 00000
# Histogram Overflow at cycle number:
# Thread 0:

Wednesday, July 15, 2015

Batch Download ESA's Sentinel satellite images

The new ESA's satellite Sentinel  data is distributed free using an open data hub.
All you need to do is to register. The data hub provide tools for searching data and creating a cart of products for downloading. Since I'm very lazy, I created a Python client that would download the data once provided with the correct data link or a product list file (that can be easily downloaded from the hub online interface. The code is freely available on GitHub at:
https://github.com/rannof/SentinelDL

Next step (if time will allow it) is to add an automated data search based on parameters such as location, data type, dates etc.

Friday, March 20, 2015

FFT using pylab

from pylab import *

data,timestep = 10*sin(arange(1000)*5/2.0/pi)+5*sin(arange(1000)*10/2.0/pi),0.01
han = hanning(len(data))
handata = han*data
n = len(data)*50
FFT = abs(fftshift(fft(handata,n)))
freq = fftshift(fftfreq(n,timestep))
plot(freq,2*FFT/sum(han))

Tuesday, January 13, 2015

ASUS EeePc fan replacement

My 2009 Asus EeePc (model 1005ha) started to make loud noise. I figured it was the fan. With an aid of this YouTube movie (see below) I was able to replace the fan and repair it. Thanks luisfilipe1983!

Assembling the tools

Opening the back

Removing the keboard

Going inside

Getting closer...

Deep inside

Fixed!


The YouTube Movie:

Wednesday, October 1, 2014

Starting a PostDoc At UC Berkeley

I'm starting A postdoc fellowship at UC Berkeley Seismological Laboratory (BSL) under the supervision of Prof. Richard Allen. For the next year or two I'll be busy working on the Earthquake Early Warning System (EEWS) algorithms. My first project will be to figure how to implement BSL EEWS algorithms for the Israeli Seismological Network. California here we come...

Tuesday, December 31, 2013

Starting to work at GII

That's it. I've finished my  2 years Post-Doc adventure at TAU.
I'm now starting to work on the Israel national Earthquake Early Warning System (EEWS).
As of tomorrow, I'm going to work at the Seismology Division of the Geophysical Institute of Israel (GII).

Tuesday, November 26, 2013

Real-Time Kernel for RaspberryPi (Take II)

After unsuccessful attempt to compile Archlinux-rt kernel,
I encountered a very good guide describing how to compile a Debian Linux-rt kernel with Xenomai for the RaspberryPi. and a completing page for creating the RPI image.
I can't say I understand it completely, but I managed to combine to two guides and compile my own RaspberryPi Linux-rt kernel.

I'm sure some of the steps are missing and things change very quickly (e.g. links, kernel versions and patches) so anyone following my steps should take that into consideration and adapt to changes.

I started by installing a fresh Debian 32bit x86 on a virtual box.
Follow the picnc guides but remember to adapt the patches and kernel version differences.

I'll try to redo the process and give here a complete guide at a later stage.

Thursday, August 29, 2013

EQMet TSA-SMA setup for SeisComp3

The following post will describe how to install EQMet INTERNET-READY STRONG MOTION ACCELEROGRAPH (TSA-SMA) and set it up for SeisComp3.


Following the instruction in the manual:
The initial steps are to connect the TSA-SMA to electricity, GPS and a computer via the USB device connection.
Then, connect to the device via serial connection.
On windows: read the manual and follow the orders using putty. Don’t forget to download the driver for the device from EQMet site (http://www.eqmet.com/eng/Linux-cdc-acm.zip)

On Linux:
install cu:
# sudo yum install uucp
get the tty of the connection:
# dmesg | grep 'tty'
on my system it was: ttyACM0 but it can be ttyS[1234] or serial.
next I connect to the device:
# cu -l /dev/ttyACM0 -s 115200
it takes a while to prompt for user and password (root,kmi) press enter if does not prompt.
I need to assign an IP based on the mac address so I check the mac:
# ifconfig
after I got the IP, it is easy to connect via http sftp and ssh.

for static IP, edit /etc/network/interfaces:
replace:
iface eth0 inet dhcp
with:
iface eth0 inet static
address <IP>
netmask <MASK>
gateway <GATEWAY>
dns-nameservers <DNS-SERVER>

connect to the device via http.
Next go to Parameters:hardware menu option.
update to your preferable network-station-location-channel ID in:
- Network ID
- Unit ID and Site ID
- dig1 ChX ID
- dig1 ChX Location code
where X in channel 1, 2 or 3.

press OK.

Next, following the response mail I got for EQmet support regarding seedlink connection:
Sir,

The limited version of Rockhound in the TSA-SMA does not support continuous recording. However, you can approximate continuous recording by adding the SEEDLink mirror module and configuring it to accept data from the RockToSlink module at address 127.0.0.1, port 18000 (or as configured). You should add this module after installing and adding in the RockToSlink module.

This module writes the received MiniSEED packets out to storage and breaks the files at approximately the specified duration (depending on MiniSEED packet contents). If you add this module, you should also consider adding an auto file delete module to avoid filling the storage. A file sender module may also be useful (SSH, FTP, or email).

The RockToSlink module for Rockhound is provided by ISTI, New York. The module and documentation is available from ISTI at http://www.isti2.com/RockToSLink/

Technical support for RockToSLink is available from ISTI at isti-info@isti.com. SEEDlink is a public domain program and protocols. Support for these programs including detailed protocol descriptions or installation and setup of the programs is not provided by Kinemetrics.

Regards,
EQMet, Technical Support
support@eqmet.com
installe the RockToSlink module and add it to the workflow:

1. Download the rocktoslik module from http://www.isti2.com/RockToSLink/dist_pub/:
> wget -v http://www.isti2.com/RockToSLink/dist_pub/RockToSLink_v1.0_dist.zip
2. Unzip it:
> unzip RockToSLink_v1.0_dist.zip
following the instractions in http://www.isti2.com/RockToSLink/current_dist/docs/RockToSLink_ovr.html:
3. sftp to the device:
> sftp root@<IP>
4. Change directory to module location:
> cd  /usr/rock/SMARTSDist/injar
5 Upload the module-update jar file:
> put RockToSLink_update.jar
6. connect via ssh:
> ssh root@<IP>
7. Change directory to module location:
> cd  /usr/rock/SMARTSDist/injar
8. Rename "RockToSLink_update.jar" to "update.jar"
> mv RockToSLink_update.jar update.jar
9. Reboot the instrument:
> reboot
Now add the RockToLink module to device layout:
10. In the device http menue Parameters: Advanced features, add options for Add/Remove Modules, Replace Modules and Advanced Modules.
11. In Module Add, add the RockToLink module.
12. Press apply changes now.

Now go to system operation and make sure all settings are correct.

enjoy.


Thursday, August 1, 2013

Machinoid Real-Time Linux for RaspberryPi

Download and install Machinoid following: http://www.machinoid.com/?p=10
apt-get install libtool gfortran
apt-get install libusb-1.0
install xenomai following the instructions in the link above.
sysctl vm.min_free_kbytes=8192
vi /etc/ld.so.conf.d/xenomai.conf
add 2 lines:
  # xenomai default configuration
  /usr/xenomai/lib
ldconfig
install wifi drivers (after I got problems with rt2800usb driver):
apt-get install firmware-ralink

create new file /etc/modprobe.d/8192cu.conf
add:
# Disable power saving
options 8192cu rtw_power_mgnt=0 rtw_enusbss=1 rtw_ips_mode=1

in /etc/dhcp/dhcp.conf
uncomment the timeout line and edit:
timeout 100;


Sunday, July 7, 2013

LSM303 accelerometer on RaspberryPi

After getting my new LSM303 accelerometer, I got my great electronics tec. Daviv Shtibelman to wire it up for RPI.
Connecting LSM303 pins 1 (SCL), 2 (SDA), 6 (GND) and 8 (V3V) to pins 5 (SCL), 3 (SDA), 6 (GND) and 1 (V3V) on the Pi board.
On my ArchlinuxARM OS I had to install i2c-tools:
> pacman -S i2c-tools
And add i2c modules:
> echo i2c-dev > /etc/modules-load.d/ i2c.conf
> echo i2c-bcm2708 >> /etc/modules-load.d/ i2c.conf
> reboot

now to check it works:
> i2cdetect -y 1
got a message like this:
     0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f
00:          -- -- -- -- -- -- -- -- -- -- -- -- --
10: -- -- -- -- -- -- -- -- -- 19 -- -- -- -- 1e --
20: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
30: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
40: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
50: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
60: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
70: -- -- -- -- -- -- -- --

Now, following this, I test it and it works. Now I'm up for creating my own code.

Made some changes to the code:

main.cpp
#include"LSM303DLHC.h"
#include<stdio.h>
#include <unistd.h>


/*
   getHeading() calculates a tilt-compensated heading.
   A float between 0 and 360 degrees is returned. You need
   to pass this function both a magneto and acceleration array.
 
   Headings are calculated as specified in AN3192:
   http://www.sparkfun.com/datasheets/Sensors/Magneto/Tilt%20Compensated%20Compass.pdf
 
*/

float getHeading(LSM303& lsm303dlhc);

#define PI 3.141592654


int main(void)
{
    uint8_t bajt;
    const char *fileN = "/dev/i2c-1";
    LSM303 lsm303dlhc(fileN);

    lsm303dlhc.enalbe();

    while(1)
    {
        lsm303dlhc.readAccelerationRaw();
        lsm303dlhc.readMagnetometerRaw();
        printf("acc [m/s^2]: \e[27;1;31m %f \e[m \e[27;1;32m %f \e[m \e[27;1;34m %f \e[m mag:  \e[27;1;31m %d \e[m  \e[27;1;32m %d \e[m \e[27;1;34m %d\e[m %fdeg\n",
               (int16_t)lsm303dlhc.acc_x_raw*0.00957,(int16_t)lsm303dlhc.acc_y_raw*0.00957,-(int16_t)lsm303dlhc.acc_z_raw*0.00957,
               (int16_t)lsm303dlhc.mag_x_raw, (int16_t)lsm303dlhc.mag_y_raw, (int16_t)lsm303dlhc.mag_z_raw,getHeading(lsm303dlhc));
        usleep(10);
       }

    }


float getHeading(LSM303& lsm303dlhc)
{
  float heading,pitch,roll,xh,yh,zh;
  // see section 1.2 in app note AN3192
  int magValue[3];
  float accelValue[3];
  magValue[0] = (int16_t)lsm303dlhc.mag_x_raw;
  magValue[1] = (int16_t)lsm303dlhc.mag_y_raw;
  magValue[2] = (int16_t)lsm303dlhc.mag_z_raw;
  accelValue[0] = (int16_t)lsm303dlhc.acc_x_raw*0.000976531;
  accelValue[1] = (int16_t)lsm303dlhc.acc_y_raw*0.000976531;
  accelValue[2] = -(int16_t)lsm303dlhc.acc_z_raw*0.000976531;

    // see appendix A in app note AN3192
  pitch = asin(-accelValue[0]);
  roll = asin(accelValue[1]/cos(pitch));

  xh = magValue[0] * cos(pitch) + magValue[2] * sin(pitch);
  yh = magValue[0] * sin(roll) * sin(pitch) + magValue[1] * cos(roll) - magValue[2] * sin(roll) * cos(pitch);
  zh = -magValue[0] * cos(roll) * sin(pitch) + magValue[1] * sin(roll) + magValue[2] * cos(roll) * cos(pitch);

  heading = 180*atan2(yh,xh)/PI;

  if (heading <0)
    heading += 360;

  return heading;
}

LSM303DLHC.cpp:
#include"LSM303DLHC.h"
#include<math.h>
#include<stdio.h>

/*Conection to Raspberry PI:
 LSM303     Raspberry PI
 VDD    ->  3V3(PIN 1)
 SDA    ->  SDA(PIN 3)
 SCL    ->  SCL(PIN 5)
 GND    ->  GND(PIN 6)
*/

#define LSM303DLHC_MAG_ADDRESS            (0x3C >> 1)
#define LSM303DLHC_ACC_ADDRESS            (0x32 >> 1)

LSM303::LSM303(const char * i2cDeviceName) : i2c_lsm303(i2cDeviceName)
{

}

uint8_t LSM303::readAccRegister(uint8_t regAddr)
{
    i2c_lsm303.addrSet(LSM303DLHC_ACC_ADDRESS);
    return i2c_lsm303.readByte(regAddr);
}

uint8_t LSM303::readMagRegister(uint8_t regAddr)
{
    i2c_lsm303.addrSet(LSM303DLHC_MAG_ADDRESS);
    return i2c_lsm303.readByte(regAddr);
}

void LSM303::writeAccRegister(uint8_t regAddr,uint8_t byte)
{
    i2c_lsm303.addrSet(LSM303DLHC_ACC_ADDRESS);
    i2c_lsm303.writeByte(regAddr, byte);

}

void LSM303::writeMagRegister(uint8_t regAddr, uint8_t byte)
{
    i2c_lsm303.addrSet(LSM303DLHC_MAG_ADDRESS);
    i2c_lsm303.writeByte(regAddr, byte);

}

void LSM303::enalbe(void)
{
   writeAccRegister(LSM303_CTRL_REG1, 0b10010111);
   writeAccRegister(LSM303_CTRL_REG4, 0b00001000);

   writeMagRegister(LSM303_MR_REG, 0x00);
}

void LSM303::readAccelerationRaw(void)
{
    uint8_t block[6];

    i2c_lsm303.addrSet(LSM303DLHC_ACC_ADDRESS);

    i2c_lsm303.readBlock(0x80 | LSM303_OUT_X_L_A, sizeof(block), block);
    acc_x_raw = (int16_t)(block[0] | (block[1] << 8)) >> 4;
    acc_y_raw = (int16_t)(block[2] | block[3] << 8) >> 4;
    acc_z_raw = (int16_t)(block[4] | block[5] << 8) >> 4;

}

void LSM303::readMagnetometerRaw(void)
{
    uint8_t block[6];

    i2c_lsm303.addrSet(LSM303DLHC_MAG_ADDRESS);
    i2c_lsm303.readBlock(0x80 | LSM303_OUT_X_H_M, sizeof(block), block);

    mag_x_raw = (int16_t)(block[1] | block[0] << 8);
    mag_y_raw = (int16_t)(block[5] | block[4] << 8);
    mag_z_raw = (int16_t)(block[3] | block[2] << 8);

}

void LSM303::readAcceleration(void)
{
    readAccelerationRaw();
}

Sunday, June 30, 2013

Real-Time Kernel for RaspberryPi

After installing and running ArchLinuxARM on my RaspberryPi, I tried to compile a real-time kernel.
With the help of moonman, following this ArchLinuxARM forum thread, I downloaded the RT patch:
https://www.kernel.org/pub/linux/kernel/projects/rt/3.6/patch-3.6.11.4-rt36.patch.gz
and the PKGBUILD:
# git clone https://github.com/archlinuxarm/PKGBUILDs.git
following this, I updated pacman:
# pacman -Syy
and installed some packages:
# pacman -S kernel26-headers file base-devel abs
next
# cd PKGBUILDs/core/linux-raspberrypi
and edit the PKGBUILD file to include the patch in the source array:

source=('config'
        'change-default-console-loglevel.patch'
        'usb-add-reset-resume-quirk-for-several-webcams.patch'
        'args-uncompressed.txt'
        'boot-uncompressed.txt'
        'imagetool-uncompressed.py'
        'https://raspy-juice.googlecode.com/svn/trunk/linux-rtc/0001-rtc-pcf8523.patch'
        'https://raspy-juice.googlecode.com/svn/trunk/linux-rtc/0002-pcf8523-i2c-register-dt.patch'
        'patch-3.6.11.4-rt36.patch')

add a patch line:
patch -Np1 -i "${srcdir}/patch-3.6.11.4-rt36.patch"

and add md5sums at the end of the PKGBUILD file. to get the correct number run:
# md5sum patch-3.6.11.4-rt36.patch

next for the build:
# makepkg --asroot -Acs

The process has failed with some rejections of patches. So we need to manually apply the patches:
first look for rejections:
# find . -name "*.rej"
then edit the file and the .rej file:
# vim XXX.c XXX.c.rej
replacing XXX with the file path and name.
Then use :split to split the screen :next to change one screen to .rej file and ctrl+ww to switch between files.
add the lines marked by + and remove lines marked by - (lines with no +- sign are used for orientation.

after manually patching all files, edit the PKGBUILD file to ignore the kernel git and the patches (adding # at the start of the line)
add a line to src/linux/security/apparmor/sid.c:
"#include <linux/cache.h>"

re-run:
# makepkg --asroot

Now for the installation part:
# pacman -U linux-headers-raspberrypi-3.6.11-11-armv6h.pkg.tar.xz linux-raspberrypi-3.6.11-11-armv6h.pkg.tar.xz

And reboot.

Well, it didn't work.

Saturday, June 1, 2013

RaspberryPi initial steps

I initiated a new project involving low-cost micro-computer RaspberryPi.
In order to begin I ordered some basic parts (costs are in USD for Israeli market, should be cheaper elsewhere):
a RaspberryPi (55$)
5v charger with microusb adapter (10$)
8GB SD card (12$)
HDMI cable (8$)

and some extra parts:
3.5" LCD screen (45$)
micro-keyboard (62$)
micro-sd low profile adapter (11.9$)
wifi dongle (18.6$)

Not so cheap but might be worth it.

after I got a couple of RaspberryPi (model B rev2 512MB RAM) devices for a project I'm working on (details about the project will come later if it will work).
I bought a SD 8GB card and hooked it to the my desktop computer.
I downloaded the ARCLinux image unzipped it and followed the instructions:
Replacing sdX with the location of the SD card, run:
dd bs=1M if=/path/to/archlinux-hf-2013-06-15.img of=/dev/sdX
In order to increase the SD space to it's full 8GB I used gparted.

upgrade system:
> pacman -Sy pacman
> pacman-key --init
> pacman -Syu
sync (3 times):
# sync
reboot:
# reboot
Now I can add packages using pacman. to look for a package use:
> pacman -Ss <package name>
then select the one you need and install by:
> pacman -S <package name>

I am currently testing the Pi with the screen and 8 AA batteries to see how long it can work. I plan to build a case with a built-in 3.5" screen and a miniature keyboard so I can have a control unit for other RaspberryPi devices. My initial thoughts of using an old Gamboy case are now switched (with the encouragement of my electronic tec. guy David Shtibelman) to build a costume made case. Of course, David is the one to build it, since I have no skills to do that.



Wednesday, January 16, 2013

Fedora 17 Linux packges

These are MY needed packages, feel free to follow my steps or ignore this.
1. Add rpmfusion to yum repo list:
 > sudo yum localinstall --nogpgcheck http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-stable.noarch.rpm http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-stable.noarch.rpm
2.
sudo yum install -y yumex tcsh gmt ImageMagick ipython scipy scitools spyder gthumb nautilus-open-terminal bluecurve-icon-theme gnome-applets gnome-themes bluecurve-gnome-theme filezilla gimp swig lshw lshw-gui gnome-tweak-tool libreoffice gv gcc gcc-gfortran gcc-c++ dconf-editor tigervnc-server
install chrome from google
install virtualbox from oracle site.
copy panels:
on old machine: dconf dump /org/gnome/gnome-panel/ > panels
on new machine: dconf load /org/gnome/gnome-panel/ < panels
change with dconf-editor:
/org/gnome/desktop/interface/icon-theme to Bluecurve
/org/gnome/desktop/wm/preferences/theme Clearlooks
/org/gnome/desktop/background/show-desktop-icons True
change user shell to tcsh: sudo usermod -s /bin/tcsh USERNAME
install microsoft fonts:
wget http://www.my-guides.net/en/images/stories/fedora12/msttcore-fonts-2.0-3.noarch.rpm
sudo rpm -ivh msttcore-fonts-2.0-3.noarch.rpm

6.Install obspy as root:
sudo easy_install -N obspy.core obspy.mseed obspy.sac obspy.gse2 obspy.imaging obspy.signal obspy.arclink obspy.datamark obspy.db obspy.earthworm obspy.fissures obspy.iris obspy.neries obspy.realtime obspy.segy obspy.seg2 obspy.seisan obspy.seishub obspy.sh obspy.taup obspy.wav obspy.xseed

Tuesday, January 1, 2013

Done with the PhD., Moving forward

Finally, I finished my PhD. and now it time to move forward.
Taking my Professors advice, I'm making the first steps in the world of seismology and I'm going to spend the next few years in Tel-Aviv University, working with Dr. Alon Ziv and Dr. Hillel Wust-Bloch.
The main work is building and maintain a local mini seismic-array network. I am going to actually place the seismometers, connect the dataloggers and build the server to collect the data. We are going to use different devices - Lennartz LE-3Dlite1DV MkII seismometers, Trillium broadband seismometers, TSA-SMA accelerometers and Summit M Hydra dataloggers. On the server side, We are going to use Seiscomp3 as our main framework, but we probably research for new algorithms and Seiscomp3 modules to better suit it to our needs.
Me getting the PhD. diploma from Prof. Rivka Carmi, presedent of BGU

Sunday, December 16, 2012

JoyWarrior24F14 Udev rules

The JW24F14 device is accessible as a HID only as root. In order to access it as a user, I had to write a UDEV rule with the help of this post. more info about udev rules can be found here.
issue the command:
> sudo vi /etc/udev/rules.d/10-JoyWarrior24F14.rules
and when in editing mode (i) add:

# Rules for accessing JoyWarrior24F14 HID as user:
SUBSYSTEM=="usb", ATTRS{idVendor}=="07C0", ATTRS{idProduct}=="0140", MODE:="666", GROUP="users"
SUBSYSTEM=="usb", ATTRS{idVendor}=="07c0", ATTRS{idProduct}=="1116", MODE:="666", GROUP="users"

Thursday, October 25, 2012

Python Event Listener - multiprocessing deamon

I wanted to have a file change listener such that if new data is added to a file I could run a code.
Following this post, I've written a modification so now the listener is working in multiprocessing.



#!/usr/bin/python
# A code to stream file as it grows
# by Ran Novitsky Nof Oct 25, 2012

import os,time
from multiprocessing import Process,Pipe

# change this file name
FileName = 'stream.txt'

# adapted from http://www.valuedlessons.com/2008/04/events-in-python.html


# the Event class take care of adding, removing and executing the functions
class Event:
    def __init__(self):
        self.handlers = set()

    def handle(self, handler):
        self.handlers.add(handler)
        return self

    def unhandle(self, handler):
        try:
            self.handlers.remove(handler)
        except:
            raise ValueError("Handler is not handling this event, so cannot unhandle it.")
        return self

    def fire(self, *args, **kargs):
        for handler in self.handlers:
            handler(*args, **kargs)

    def getHandlerCount(self):
        return len(self.handlers)

    __iadd__ = handle
    __isub__ = unhandle
    __call__ = fire
    __len__  = getHandlerCount
 
# the stream class takes care of the
# function to fire in case of an event
class Stream:
  def __init__(self,fname):
    self.fname = fname
    self.fd = open(fname,'r')
    self.get_stream()
  def get_stream(self):
    # this is the actual function to fire
    print self.fd.read(),
  def restart(self):
    self.fd.seek(0)
 
# the MockFileWatcher creats the multiprocessing deamon to alert on file change
class MockFileWatcher:
    def __init__(self,source_path,sleeptime=0.1):
        self.fileChanged = Event()
        self.source_path = source_path
        self.running=False
        self.deamon=False
        self.fd = None
        self.sleep = sleeptime
        # the next two lines can be set outside the class
        self.stream = Stream(source_path)
        self.fileChanged += self.stream.get_stream # add event handler
    def watchFiles(self):
        # open the file in non-blocking read mode
        self.fd = os.open(self.source_path,os.O_RDONLY | os.O_NONBLOCK)
        # go to the end
        os.lseek(self.fd,0,2)
        self.running=True
        self.deamon=False
        while self.running:
          # try to read
          if os.read(self.fd,1)!='':
            # New data was added
            os.lseek(self.fd,0,2) # go to the end of file
            self.fileChanged() # fire the event
          # see if the parent process has a message
          if self.chiled_conn.poll():
            # execute the message
            eval(self.chiled_conn.recv())
          # sleep for a while
          time.sleep(self.sleep)
    def watchFilesDeamon(self):
        # start the deamon
        self.parent_conn,self.chiled_conn = Pipe()
        p = Process(target=self.watchFiles)
        p.start()
        self.p = p  
        self.deamon=True
        return p
    def stop(self):
        # stop deamon/process
        if self.deamon:
          self.parent_conn.send('self.stop()')
          print "Hold while stopping child process..."
          # hold depends on the sleeping time
          if self.parent_conn.recv():
            pass
            #print "chiled is stopped"
          self.p.terminate() # terminate multiprocessing
          self.deamon=False
        # stop process
        self.running=False
        self.fd=None
        # if this is the process make sure parent know we are done
        if not self.deamon:  self.chiled_conn.send(True)      
        return True

if __name__=='__main__':
  watcher = MockFileWatcher(FileName)
  p = watcher.watchFilesDeamon()
  print 'from this point the listner is running at the background'
  print 'add more code here so you can see how it works.'

Try this by changing the file name at the top and run it. then add new lines to the file.

Wednesday, October 24, 2012

SeisComp3 - get events data

After installing  Seiscomp3 and running it for a while, now it is time to play with the data.
As a first step I want to be able to see the events outside the SeisComp3 environment.
This will later serve me when exporting the data to a web page or reports.
The steps described here are: 1) get a list of events, 2) export the data to an xml file and 3) print the data as a bulletin.
1) Get a list of events:
scevtls -d mysql://sysop:sysop@localhost/seiscomp3
Will give a list of event codes like:

Wrong 'begin' format -> setting to None
Setting start to 1970-01-01 00:00:00
Setting end to 2012-10-24 08:17:13
gfz2012uizn
gfz2012atlf
...
where gfz will be the agency prefix and the last 4 letters will be random code.
You can also set begin and end time to get only a time window events. see scevtls documentation.

2) For this example I select one event (in csh environment):
set EVENT = gfz2012uizn
Now we can export the event data to an xml file:
scxmldump -d mysql://sysop:sysop@localhost/seiscomp3 -E $EVENT -P -M -A -o test.xml
see scxmldump for more information

3) Print the data to stdout:
scbulletin -i test.xml
Note that it is possible to skip the xmp part by using:
scbulletin -d mysql://sysop:sysop@localhost/seiscomp3 -E $EVENT
also adding -3 as a flag will output even more details.
see scbulletin for more info.

I'll update later with a python script to export the data to a Google Earth kml file.

Monday, September 24, 2012

Installing SeisComp3 on fedora 17

After installing Fedora 17  I will now describe how to install MySQL server and SeisComp3.

first, install MySQL (following this):
> sudo yum install -y mysql mysql-server

enable and start MySQL service:
> sudo systemctl enable mysqld.service
> sudo systemctl start mysqld.service

Next, secure the server:
> sudo /usr/bin/mysql_secure_installation
(answer yes to all questions and change the MySQL root password)
If you need to change the database location from the default follow this (source):
Stop the service:
> systemctl stop mysqld.service
Copy from default to new location:
> cp -R -p /var/lib/mysql /path/to/new/datadir
Clean unneeded files:
> rm /path/to/new/datadir
(don't worry about error messages - the undeleted files are the databases)
Edit the configuration file:
>  gedit /etc/mysql/my.cnf
(change keyword value of datadir to the new location)
Restart the service:
> systemctl restart mysqld.service

Now to the SeisComp3 install (Zurich version for fedora16 modified for fedora 17):
Add needed packages:
> sudo yum install -y boost-filesystem boost-iostreams boost-program-options boost-serialization boost-signals
boost-thread boost-wave boost-regex boost-devel ncurses python-numeric numpy alsa-utils festival
Link installed libraries (so.1.48) to the needed packages (so.1.47):

> sudo ln -s /usr/lib64/libboost_iostreams.so.1.48.0 /usr/lib64/libboost_iostreams.so.1.47.0

> sudo ln -s /usr/lib64/libboost_filesystem.so.1.48.0 /usr/lib64/libboost_filesystem.so.1.47.0
> sudo ln -s /usr/lib64/libboost_system.so.1.48.0 /usr/lib64/libboost_system.so.1.47.0
> sudo ln -s /usr/lib64/libboost_regex.so.1.48.0 /usr/lib64/libboost_regex.so.1.47.0
> sudo ln -s /usr/lib64/libboost_thread-mt.so.1.48.0 /usr/lib64/libboost_thread-mt.so.1.47.0
> sudo ln -s /usr/lib64/libboost_program_options.so.1.48.0 /usr/lib64/libboost_program_options.so.1.47.0
> sudo ln -s /usr/lib64/libboost_signals.so.1.48.0 /usr/lib64/libboost_signals.so.1.47.0
Put key files in ~/.seiscomp3/key/ directory
Install using the setup and seiscomp config procedures described in the SeisComp3 documentations
add a source to env.sh file in your .bashrc:
source ~/seiscomp3/lib/env.sh

For the seattle release use:
>sudo yum install -y boost boost-devel python-numeric


Don't forget to download and extract to the seiscomp directory, the correct fedora seiscomp package (e.g. seiscomp3-zurich-xxxx.xxx.xx-fedora16-x86_64.tar.gz), the maps (e.g. seiscomp3-maps.tar.gz) and the methadata of GE network (e.g. seiscomp3-metadata-GE-2010.tar.gz).

Tuesday, August 28, 2012

Howto write to stdout XY

how to use python to write to a certain location (x,y) on the stdout (terminal):
copied form here
import sys
def print_there(x, y, text):
     sys.stdout.write("\x1b7\x1b[%d;%df%s\x1b8" % (x, y, text))
     sys.stdout.flush()

Creating a software raid

1. format the hard drives to raid using fdisk (gdisk in fedora 17)
or with parted (for 3TB disks):

> parted /dev/sdx
        (parted) mklabel gpt
        (parted) unit TB
        (parted) mkpart primary 0 3001G
        (parted) print
        (parted) quit

2. Create the raid device (change level and devices as needed):
mdadm --create --level=1 --raid-devices=2 /dev/md0 /dev/sd[bc]1
3. format the file system of the new device:
mkfs.ext4 /dev/md0
4. create the mdadm.conf file:
mdadm --detail --scan > /etc/mdadm.conf
5. add mail alert:
echo "MAILADDR your_mail@somthing.com" >> /etc/mdadm.conf
echo "MAILFROM your_server" >> /etc/mdadm.conf
6. test mail alert:
mdadm --monitor --scan --test --oneshot
7. start mail alert deamon:
systemctl enable mdmonitor.service
systemctl start mdmonitor.service
8. edit the mail aliases file to forward root messages to your email:
vi /etc/aliases
add your mail under the line:  # Person who should get root's mail
run newaliases command
see also at:
http://www.iceteks.com/articles.php/linuxmdadmraid/

* after name change, create the /etc/mdadm.conf file and run:
sudo update-initramfs -u
or
sudo dracut -f /boot/initramfs-[current].img

Monday, August 27, 2012

Installing Fedora Core 17

based on my-guides post-install post

1. Add to sudo:
> su --login -c 'visudo'
add under the line:
root    ALL=(ALL)       ALL
the line:
username ALL=(ALL)      ALL
and remove the sign # from the line with %wheel
save and test using the line:
> sudo whoami
(answer should be root)
2. Disable SeLinux and firewall:
> sudo vi  /etc/selinux/config
change enabled to disabled
> sudo systemctl disable iptables.service
> sudo systemctl stop iptables.service
3. change autologin for user:
sudo vi /etc/gdm/custom.conf
under [daemon], add:
TimedLoginEnable=true
TimedLogin=yourUserName
TimedLoginDelay=1
4. Switch to gnome3 fallback mode:
> gsettings set org.gnome.desktop.session session-name 'gnome-fallback'
5. enable sshd:
>sudo systemctl enable sshd.service
> sudo systemctl start sshd.service

6. auto rotate screen on login:
add to .profile on home:
xrandr --output VGA-1 --rotate left
7.add daily update report (as root):
following this:
> vi /etc/cron.daily/yum_update_note
#!/bin/sh

if [ ! -f /var/lock/subsys/yum ] ; then
        msg=`/usr/bin/yum check-update 2>&1`
        i=`echo "$msg" | wc -l`
        if ((i>1)); then
                echo "$msg" | /bin/mail -s "yum check-update output" root
        fi
fi

and chnage mode for executable:

> sudo chmod +x /etc/cron.daily/yum_update_note


Thursday, May 10, 2012

Howto run a process automatically when a certain mail is downloaded by Thunderbird

In order to run a file whenever a certain mail arrives to my inbox I created a filter using the Thunderbird add-on FiltaQuilla. The add-on expand the messages filtering options and add an option to "run a file". when a certain mail match the filter, Thunderbird will run the file specified. I use a python code - code.py so in the filter configuration under Linux (Fedora core 14) I use this line:

/usr/bin/python,/home/UserName/code.py,-a

The line is a comma separated 3 parts line. The first part is the python interperter, the second is the script name to run and the last part is an argument needed for the script to run. Omitting the first part and the filter wont work.

Monday, November 21, 2011

HowTo Mosaic ASTER Global Digital Elevation Model (GDEM)

A second version of ASTER GDEM was released in Oct. 17 2011 by The Ministry of Economy, Trade and Industry of Japan (METI) and the National Aeronautics and Space Administration (NASA). The GDEM is in a 30m pixel resolution and 1x1 degree tiles. The data is distributed as zipped GeoTIFF files. When a larger than 1x1 deg. DEM is needed for Interferometry processing, you can use my GDEM.py script available from my script page. This python script will need a system with pylab and PIL python modules.

Simply:
Download the GDEM zipped files (see here on how to do that).
Unzip the zip files in the working directory.
The working directory should contain sub-directories containing the GeoTIFF files.
Run the script giving -h for help or the final DEM name as arguments.
The Final DEM is aimed for using as DEM in ROI-PAC interferometry processing.
If a -g is added as an argument, the produced DEM will be suitable for interferometric processing using Gamma software.

Saturday, July 30, 2011

Create a photo collage using a single command-line

In order to create a contact-sheet or a collage of many files under linux, I prefer to use a single command-line. I create a directory with all the images and use ImageMagick montage function:
montage -font Arial-Narrow -pointsize 10 -label '%t' -resize 100x150 *.jpg -geometry +3+3 -tile 4x4 -shadow collage.jpg

Thursday, July 28, 2011

HowTo use a computer without a screen attached

The Problem:
I have a Linux machine (Fedora 14)  with no screen attached. I view the graphics via VNC server. I had to adjust the system to use the Nvidia driver and tweak it for no screen.

The solution:
First we need to install the Nvidia driver. Then, we need to adjust the X configuration for no screen and our preferred resolution.

for more details see: http://forums.fedoraforum.org/showthread.php?t=204752

1. Update the kernel:

su
yum update kernel*
reboot


2. Add rpmfusion to your repository.

su
rpm -Uvh  http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-stable.noarch.rpm   http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-stable.noarch.rpm

3. Depending on your card add nvidia akmod to the kernel and the proper driver. for a Gforce 9500GT on my 64bit machine:
yum install akmod-nvidia-260.19.36-1.fc14.x86_64 xorg-x11-drv-nvidia-libs-260.19.36-1.fc14.x86_64

4. Edit the xorg.conf file. don't forget to  adjust the Nvidia card for no screen attached by adding:
 Option "ConnectedMonitor" "CRT"
to the device section and adjust the screen resolution under the Screen section:

Section "Device"
    Identifier     "Card0"
    Driver         "nvidia"
	Option "ConnectedMonitor" "CRT"
EndSection

Section "Screen"
    Identifier     "Screen0"
    Device         "Card0"
    Monitor        "Monitor0"
    SubSection     "Display"
        Viewport    0 0
        Modes      "1280x1024"
    EndSubSection
EndSection

Thursday, July 14, 2011

Daichi - ALOS : finding Images and Baseline Calculation

Using JAXA satellite "DAICHI" (ALOS) for SAR Interferometry might raise the need for calculating perpendicular baselines. One can use AUIG site to locate the needed images for interferometry and use PALSAR baseline calculator CalcBP for calculating perpendicular baselines. more info is available here.

Thursday, March 31, 2011

Installing Linux Fedora Core 14

Well, after a long time using Fedora 11 I'm moving on to Fedora 14.
Here I'll detail the steps I used for the installation.
I generally follow my Fedora 10 installation guide with some changes of course.

LiveCD:
I use a liveCD (download page) to install.
after loading the system via cd,
I use the "Install to Hard Drive" icon.
next,next,next.
Giving the Hostname, setting location, giving a good root password.

File System:
Since I use several hard drives I need to configure the file system. I use one 1.5TB disk as my home, another one for data and processing and one for the system so all the installation will only deal with the last one. later I'll mount all the rest of the disks.
so the type of installation for me is "Replace Existing Linux System(s)", but with "Review and modify layout" switch on and I select the system disk as "Install Target Devices".
Next I change the partitions and mount points of all disks as needed.
Finally, The boot loader is set up and the system is installed on the hard drive.

Updates and softwares:
As a first step few softwares should be installed and update (this might take forever):
system-config-network
yumex
gconf-editor
gedit-plugins
gthumb
nautilus0open-terminal
tango-icon-theme
tango-icon-extras
fonts for Hebrew
all stix fonts
cm-lgc fonts
blender
gimp
gv
ImageMagick
inkscape
filezilla
thunderbird
emacs
open office
vim-enhanced
gcc
gmt
grace
HDF
HDF5
lapack
netcdf
numpy
python support for open office
compat-gcc
glade3
ipython
pango-devel
wget
tcsh
system-config-samba
system-config-users
system-config-nfs
dos2unix
selinux configuration gui
matplotlib
scipy
scitools
kdocker
numlockx
bluecurve icons
ssmtp

Customizing:
remove SElinux policy
disable filrewall
configure remote desktop
add user to sudo:

su --login -c 'visudo'
Now go below the line:
root ALL=(ALL) ALL
press a, and type the following
username ALL=(ALL) ALL
where username is the username you use to login. Next press Escape.
Now, if you want to be prompted for your root password each time you use the sudo command go to this line:
# %wheel ALL=(ALL) ALL
and with your cursor pointing on # press x
If you don't want to be prompted for your root password each time you use the sudo command go to this line:
# %wheel ALL=(ALL) NOPASSWD: ALL
and with your cursor pointing on # press x
Next, press :wq to save and exit. (type the : as well)
* Adding your user to the wheel group
Just type:
su -c 'gpasswd -a username wheel'
* Testing sudo
To test if you have done it correctly as a simple user type:
$ sudo whoami
If everything is working properly the command should return the word 'root'.

change autologin for user:
sudo vi /etc/gdm/custom.conf
under [daemon], add:
AutomaticLoginEnable=true
AutomaticLogin=yourUserName

edit /etc/fstab for mounting external HD (LAN):
//XXX.XX.XX.XX/DATA /mountpoint cifs auto,owner,rw,username=XXX,uid=XXX,gid=XXX 0 0
NOTE! - it will not mount on startup. need to do: "sudo mount mountpoint" to mount. one can add password=[user-passwd] after username=[username] or instead username and password : credentials=[path to a file containing user and password]
the file should contain the remote system properties:
username=[username]
password=[userpassword]

Enable Num Lock on GNOME startup:
su -c 'yum install numlockx'
su -c 'cp /etc/gdm/Init/Default /etc/gdm/Init/Default_backup'
su -c 'gedit /etc/gdm/Init/Default'
And add the following lines at the end of the file before the exit 0.
if [ -x /usr/bin/numlockx ]; then
/usr/bin/numlockx on
fi

set the samba server

set the mail server:
sudo vi /etc/ssmtp/ssmtp.conf
change the line:
mailhub=mail
to:
mailhub=[your outgoing mail server]

Set the screen resolution for vnc and no monitor:
create xorg.conf file
sudo Xorg :1 -configure
sudo cp /root/xorg.conf.new /etc/X11/xorg.conf
and add lines to the xorg.conf file:

Section "Monitor"
Identifier   "Monitor0"
ModelName    "LCD Panel 1280x1024"
HorizSync    31.5 - 64.0
VertRefresh  56.0 - 65.0
Option    "dpms"
EndSection



Section "Device"
        ### Available Driver options are:-
        ### Values: <i>: integer, <f>: float, <bool>: "True"/"False",
        ### <string>: "String", <freq>: "<f> Hz/kHz/MHz",
        ### <percent>: "<f>%"
        ### [arg]: arg optional
        #Option     "SWcursor"           # [<bool>]
        #Option     "HWcursor"           # [<bool>]
        #Option     "NoAccel"             # [<bool>]
        #Option     "ShadowFB"           # [<bool>]
        #Option     "UseFBDev"           # [<bool>]
        #Option     "Rotate"             # [<str>]
        #Option     "VideoKey"           # <i>
        #Option     "FlatPanel"           # [<bool>]
        #Option     "FPDither"           # [<bool>]
        #Option     "CrtcNumber"         # <i>
        #Option     "FPScale"             # [<bool>]
        #Option     "FPTweak"             # <i>
        #Option     "DualHead"           # [<bool>]
Identifier  "Card0"
Driver      "vesa"
BusID       "PCI:1:0:0"
EndSection

Section "Screen"
Identifier "Screen0"
Device     "Card0"
Monitor    "Monitor0"
SubSection "Display"
Viewport   0 0
Depth     24
Modes "1280x1024"
EndSubSection
EndSection




set the panels and desktop background as you like it.

Install RPMfusion repositories following: www.rpmfusion.org/Configuration

add true type fonts from windows

Wednesday, October 13, 2010

RoiView - Explore InSAR data and more

RoiView was originaly designed to replace DGX/MDX software which require bureaucracy to obtain. This software was used in order to view ROI-PAC interferometry processing results but producing an image from file was not so strait forward. In time support was added for ERDAS ER Mapper header files and for user provided parameters with or without header files.

Current RoiView version (0.75) is available for download at SourceForge.net. Fast, secure and Free Open Source software downloads
See also RoiView's help files

Screenshot:

Tuesday, September 28, 2010

HowTo Parse HTML text using python.

I needed to to get the text of a specific div elemnt in an html file. I tried to use python's standard library modules for markup processing such as htmllib etc. but I couldn't figure how to use them. I've created my own module just for getting the text from an html element:

#!/bin/env python
from htmlentitydefs import entitydefs as ent
import string
# This module enable you to extract text from a certian HTML element
#
# by Ran Novitsky Nof, 2010
# ran.nof@gmail.com
#
# example of use:
# say we want to get the text in an element of type tag (e.g. 'div','a','span' etc.)
# who has an attribute key (e.g. "id","class","href" etc.) with a value of val
# for example in order to extract the text of a div element with id of textdiv from a file htmlfile.html:
#
# <html>
# <head>
#   :
# </head>
# <body>
#   :
# <div "id"="textdiv">I will not buy this <a href="spam">record</a> it is scratched. </div>
#   :
# </body>
# </html>
#
# use:
# from htmlparser import Parser
# htmlfile='htmlfile.html'
# tag,key,val = ('div','id','textdiv')
# text=Parser(htmlfile).getText(tag,key,val)
# print(text)
#
class Element():
  def __init__(self):
    self.startTag = -1
    self.endTag = -1
    self.attrib = {}
    self.keys = self.attrib.keys()
    self.innerHTML = ''
    self.tag = ''
    self.start = -1
    self.end = -1

class Parser():
  def __init__(self,infile):
    self._root = open(infile).read()
    self._root = self._root[self._root.find('<body'):self._root.find('</body')]
    self._root = self._root[self._root.find('>')+1:]
    self.tags = set()
    i,j=0,0
    self.tagstarts = {}
    self.tagends = {}
    self.elements = []
    while i<len(self._root): 
      i = self._root.find('<',i)
      j = self._root.find('>',i)
      if not j>i: break
      tag = self._root[i+1:j].split()[0]     
      if tag.startswith('/'):
        tag = tag[1:]
        self.tagends[tag][-1-self.tagends[tag][::-1].index(None)]=((i,j))
      else:
        self.tags.add(tag)
        if tag in self.tagstarts:
          self.tagstarts[tag].append((i,j))
          self.tagends[tag].append(None)
        else:
          self.tagstarts[tag]=[(i,j)]
          self.tagends[tag]=[None]
      i=j+1
    self.getElements()
  def getElements(self):
    for tag in self.tags:
      if not tag in ['img']:
        for i in range(len(self.tagstarts[tag])):
          element = Element()
          element.startTag = self.tagstarts[tag][i][0]
          element.endTag = self.tagstarts[tag][i][1]
          tagData = self._root[element.startTag+1:element.endTag].replace("\"","")
          element.tag = tag
          element.attrib=dict([a.split('=') for a in tagData.split() if '=' in a])
          element.start = element.startTag
          element.end = self.tagends[tag][i][1]
          element.innerHTML = (self.tagstarts[tag][i][1]+1,self.tagends[tag][i][0])
          self.elements.append(element)
  def getText(self,tag,key,val):
    element = [element for element in self.elements if element.tag==tag and element.attrib[key]==val]
    if len(element):
      element = element[0]
      start,end = element.innerHTML
      text = self._root[start:end]
      i,j=0,0
      while i<len(text):
        i  = text.find('<',i)
        j = text.find('>',i)
        if i<0 or j<0: break
        text = text[:i]+text[j+1:]
      i,j=-1,-1   
      while i<len(text):
        i  = text.find('&',i+1)
        j = text.find(';',i+1)
        if i<0 or j<0: break
        if text[i+1:j] in ent.keys(): text = text[:i]+ent[text[i+1:j]]+text[j+1:]
    else:
      text=None 
    return text
Note the code does not check if the html code is correct. it also work only for the body part, and ignore img tags.
   

Sunday, July 18, 2010

SSH with no password

When you wish to login to a remote system through an ssh session without using any password you'll need to produce a key on the local computer and on the remote computer in order to allow password - free entrance.
first you'll need to produce the key in the local computer:
ssh-keygen -t dsa -b 1024 -P ""
press Enter on the questions followed by the command.
the output will be something like:
Your identification has been saved in /home/yourname/.ssh/id_dsa.
Your public key has been saved in /home/yourname/.ssh/id_dsa.pub.
The key fingerprint is:

4f:9e:4c:17:8f:19:37:32:bc:45:4f:2a:a2:8e:1c:15 yourname@your.loacl.machine
The key's randomart image is:
+--[ DSA 1024]----+
|           .o    |
|           .oo.  |
|          . o+   |
|      o    *.+. .|
|       .E . +  . |
|      o  .+.. . .|
|       +o.o. +   |
|       o+E       |
|       o=.       |
+-----------------+


This will produce two files in you .ssh directory: id_dsa and id_dsa.pub
cat id_dsa.pub into ~/.ssh/authorized_keys on the remote system you want to log into.
change mod to 600:
chmod 600 ~/.ssh/authorized_keys
(source of the tip)

Friday, June 25, 2010

Howto export points to a Google Earth KML file

The problem:
A table of xyz data points can be viewed spatially with different methods:
GMT and ArcView are just a couple of options. I often get tables of spatial data points in Microsoft Excel (xp ver.) format or CSV of some kind. An easy (and free) way to visualize the data points on Windows would be to export the data and view it on Google Earth.
The solution:
points2kml (115kb).
In order to ease the work-flow I wrote a VBA macro add-in for Excel that exports selected fields (longitude, latitude, name and data) to a Google Earth KML file. the point2kml add-in and can be found on my scripts page.

Wednesday, June 2, 2010

How to format a hard drive on Linux

The Problem:
How to format a hard drive on Linux.

The Solution:
in order to format a HD on Linux you'll need super user privileges.
you can use sudo if you have sudo privilages or in the terminal
switch to super user:

>su
(give super user pwd)
and ignore the sudo in the following commands. if you're system can't find the commands while in su mode, try to add /sbin/ before the command.
follow the steps below: marked with •
run commnads marked with >
commands output looks like this.
Warning: formatting your HD will erase all data on it!
  • Connect the HD to the computer, make sure it's on. 
  • get the hard drive's (device) name on your system:
>df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/mapper/vg_insarlablin-lv_root
                      908G   77G  786G   9% /
/dev/sda1             194M   31M  153M  17% /boot
/dev/sdb1             1.4T  1.1T  193G  86% /home/novitsky
/dev/sdc1             1.4T  1.1T  193G  86% /home/novitsky/data

Now lets say our HD is sdb1.
  • lets build it's partition:
>sudo fdisk /dev/sdb
The number of cylinders for this disk is set to 182401.
There is nothing wrong with that, but this is larger than 1024,
and could in certain setups cause problems with:
1) software that runs at boot time (e.g., old versions of LILO)
2) booting and partitioning software from other OSs
   (e.g., DOS FDISK, OS/2 FDISK)


Command (m for help): 
  • pressing m will get this menu:

Command action
   a   toggle a bootable flag
   b   edit bsd disklabel
   c   toggle the dos compatibility flag
   d   delete a partition
   l   list known partition types
   m   print this menu
   n   add a new partition
   o   create a new empty DOS partition table
   p   print the partition table
   q   quit without saving changes
   s   create a new empty Sun disklabel
   t   change a partition's system id
   u   change display/entry units
   v   verify the partition table
   w   write table to disk and exit
   x   extra functionality (experts only)

  • pressing p will print the current partition:

Disk /dev/sdb: 1500.3 GB, 1500301910016 bytes
255 heads, 63 sectors/track, 182401 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk identifier: 0x000923e6

   Device Boot      Start         End      Blocks   Id  System
/dev/sdb1               1      182401  1465136001   83  Linux

  • pressing d will delete the current partition. depending on the partition you want to delete (this case 1) press the number of the partition to be deleted. if all the disk should be formatted and there are several partitions repeat pressing d and the number of the partitions until you delete them all.
  • pressing n will create a new partition.
  • pressing p will create a primary partition. select 1 as the number of the partition. use the defaults (just press Enter) for the start and end cylinders.
  • pressing t will set the file system type.
  • pressing l will list the available file systems.
  • select 83 if you want a Linux file system
  • pressing w will write and exit to shell.
Now we will create a new file system on the new partition:
> sudo mkfs -t ext2 /dev/hdb1
remember to change the number at the end if your HD device number is different.
if your using different fs than ext2 (newer Linux systems have ext3 and fc11 has ext4) replace the ext2 in the command with the correct fc name.
the mkfs may take some time to run (depending on the computer and HD size).

To add a label to the partition use:
e2label /dev/sdb1 NEWLABEL
where sdb should be changed to the correct device and NEWLABEL changed to the new label.

Now add the device to the fstab file in order to mount it:
> sudo vi /etc/fstab
and add a line like:
LABEL=NEWLABEL MOUNTPOINT ext3 defaults 1 2
where NEWLABEL is the disk labe and MOUNTPOINT is where you want to mount the disk.
mount it using:
> sudo mount -a
Note you might need to change permissions and owner of the mount point:
> sudo chown username:usergroop MOUNTPOINT
when finnished - reboot.