Thursday, May 18, 2017

Send Large attachments with google API for gmail

The problem: How to send large size images using google's gmail API for python. The documentation is not very good and a 10MB size limit is not mentioned. Following their quickstart example, I use body-media instead of body to send large files.
See below for an example:
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import logging
log = logging.getLogger("log")
import httplib2from StringIO import StringIO
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/gmail-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'# mail size limit. can get up to 35MB
MAILSIZELIMIT = 1024*1024*25 # set to 25MB
def get_credentials():
    """Gets valid user credentials from storage.

    If nothing has been stored, or if the stored credentials are invalid,
    the OAuth2 flow is completed to obtain the new credentials.

    Returns:
        Credentials, the obtained credential.
    """
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'gmail-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_known_args()
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        log.debug('Storing credentials to ' + credential_path)
    return credentials

def jpg2attachment(filename):
  """Creates an image message from file on disk.
       See other examples here: https://developers.google.com/gmail/api/guides/sending"""
  with open(filename,'rb') as f:
    img = MIMEImage(f.read())
    img.add_header('Content-Disposition', 'attachment', filename=os.path.split(filename)[1])
  return img

def send_gmail_with_attachments(sender,recipients,subject,txtcontent,attachmentfilelist=[]):
  """Sends email with attachments.
       sender: string of sender name/address
       recipients: a list of strings with recipients emails
       subject: string of the subject
       txtcontent: string with textual message body
       attachmentfilelist: a list of file names. in this example it is only for jpg files.
  """
  log.info('Sending email to: {}'.format(recipients))
  log.debug('Connecting to gmail server')
  try:
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
  except:
    log.error('Error with credentials')
    return False
  for i in range(3): # 3 attempts to connect to service.
    try:
      service = discovery.build('gmail', 'v1', http=http)
      break
    except httplib2.ServerNotFoundError,e:
      if i==2:
        log.error('{} of 3 attempts to connect to mail server failed.'.format(i+1))
        log.error(e)
        return False  
  COMMASPACE = ', '
  toaddrs = COMMASPACE.join(recipients) # get recipients list
  # Create the container (outer) email message.
  msg = MIMEMultipart()
  msg['Subject'] = subject
  msg['From'] = sender
  msg['To'] = toaddrs
  # add content
  txt = MIMEText('text', 'plain')
  txt.set_payload(txtcontent)
  msg.attach(txt)
  raw = msg.as_string()
  attachments = [jpg2attachment(a) for a in attachmentfilelist if a.endswith('jpg')]
  # send the message via gmail
  attempt = 1
  while len(attachments): # as long as we have attachements
    mediafile = StringIO() # placeholder for media body file
    while len(attachments) and len(raw)+len(attachments[0].as_string())<MAILSIZELIMIT: # make sure we are not exceeding size limit.
      msg.attach(attachments.pop(0)) # remove image attachment from list and put in message
      raw = msg.as_string() # serialize message
    mediafile.write(raw) # populate the media body file
    media = MediaIoBaseUpload(mediafile,mimetype='message/rfc822',chunksize=1024*1024,resumable=True) # pack as a media message
    if len(msg.get_payload())>1: # make sure message is not empty (in case a single attachment is over size limit
      try:  
        service.users().messages().send(userId='me',body={},media_body=media).execute() # send the message
      except Exception,e:
        log.error("attampt {}. Can't send email.\n\n{}\n\n".format(attempt,e))
        if attempt<=3:
          attempt+=1
        else:
          figures = [img.get_filename() for img in msg.get_payload() if img.get_content_maintype()=='image']+[img.get_filename() for img in attachments]
          log.error('Failed to send figures: {}'.format(figures))
          return
      else:
        sentimages = [img for img in msg.get_payload() if img.get_content_maintype()=='image']
        log.info('Sent figures: {}'.format([img.get_filename() for img in sentimages]))
        [msg.get_payload().remove(img) for img in sentimages]# remove figures from message
    elif len(attachments): # there is a too large image, send to last or remove.
      img = attachments.pop(0)
      if not len(img.as_string())>MAILSIZELIMIT:
        log.debug('Large size attachment {} ({}) pushed to end of line.'.format(img.get_filename(),len(img.as_string())))
        attachments.append(img) # push to end of line
      else:
        log.error('Failed to send figure: {}, size: {}'.format(img.get_filename(),len((img.as_string()))))
  return True

Wednesday, January 6, 2016

Real-Time Kernel for RaspberryPi 2

This post will describe how to compile a RT-kernel for RaspberryPi 2.
I'm using UBUNTU 14.04 LTS 64bit for compiling Raspbian Jessie.
After a long time, I needed a RT kernel for a new project with the new RaspberryPi2. Realizing My old post has broken links, I struggled again with the task. This time, after a successful attempt, I'm determined to put it all here step by step.

##### Tested on UBUNTU 14.04 LTS with Rasbian Jessie 4.1
# install needed packages for compiling the kernel
sudo apt-get install libncurses5-dev libncursesw5-dev
#Download Raspberry Pi tools:
git clone https://github.com/raspberrypi/tools.git
# Download source files and patches
git clone -b rpi-4.1.y --depth 1 git://github.com/raspberrypi/linux.git linux-rpi-4.1.y
# Download RT patch 
wget https://www.kernel.org/pub/linux/kernel/projects/rt/4.1/patch-4.1.15-rt17.patch.xz
xz -d patch-4.1.15-rt17.patch.xz
# patch the kernel. This step might produce errors
# if rt-patch and the kernel don't match
cd linux-rpi-4.1.y
cat ../patch-4.1.15-rt17.patch | patch -p1
#Export the following variables to specify cross-compilation options:
export ARCH=arm 
#to compile on a x64-machine you need to export:
export CROSS_COMPILE=~/tools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64/bin/arm-linux-gnueabihf-
#create a config file for Raspberry Pi 2:
make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm bcm2709_defconfig
# make changes to the config
make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm menuconfig
enable Preemption model -> Fully preemptible Kernel (RT)
change timer freq to 1000Hz
enable device tree (all options)
#Compile the kernel:
make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm -j5
#Install modules, will result in "lib" folder with modules and firmware:
mkdir kernel-rt
INSTALL_MOD_PATH=kernel-rt make CROSS_COMPILE=${CROSS_COMPILE} ARCH=arm modules_install
## Install the kernel to non-RT image:
Copy arch/arm/boot/Image to /boot/kernel7.img on SD card.
Copy arch/arm/boot/dts/bcm2709-rpi-2-b.dtb to /boot/bcm2709-rpi-2-b.dtb on SD card
# add to /boot/config.txt
echo “device_tree=bcm2709-rpi-2-b.dtb” >> /boot/config.txt
Copy (merge if necessary) kernel-rt/lib to / on SD card.

## Testing RT-Image
Insert SD card and connect RPI to power, keyboard and screen. 
Login with user pi and password raspberry
#Installing cyclictest utility on Raspberry Pi:
git clone http://git.kernel.org/pub/scm/linux/kernel/git/clrkwllms/rt-tests.git 
cd rt-tests
make all
cp ./cyclictest /usr/bin/
cd ~
sudo cyclictest -l100000 -m -n -a0 -t1 -p99 -i400 -h400 -q

# WITH NO RT RESULT:
# Total: 000100000
# Min Latencies: 00015
# Avg Latencies: 00021
# Max Latencies: 00118
# Histogram Overflows: 00000
# Histogram Overflow at cycle number:
# Thread 0:

# WITH RT RESULT:
# Total: 000100000
# Min Latencies: 00013
# Avg Latencies: 00020
# Max Latencies: 00066
# Histogram Overflows: 00000
# Histogram Overflow at cycle number:
# Thread 0:

Wednesday, July 15, 2015

Batch Download ESA's Sentinel satellite images

The new ESA's satellite Sentinel  data is distributed free using an open data hub.
All you need to do is to register. The data hub provide tools for searching data and creating a cart of products for downloading. Since I'm very lazy, I created a Python client that would download the data once provided with the correct data link or a product list file (that can be easily downloaded from the hub online interface. The code is freely available on GitHub at:
https://github.com/rannof/SentinelDL

Next step (if time will allow it) is to add an automated data search based on parameters such as location, data type, dates etc.

Friday, March 20, 2015

FFT using pylab

from pylab import *

data,timestep = 10*sin(arange(1000)*5/2.0/pi)+5*sin(arange(1000)*10/2.0/pi),0.01
han = hanning(len(data))
handata = han*data
n = len(data)*50
FFT = abs(fftshift(fft(handata,n)))
freq = fftshift(fftfreq(n,timestep))
plot(freq,2*FFT/sum(han))

Tuesday, January 13, 2015

ASUS EeePc fan replacement

My 2009 Asus EeePc (model 1005ha) started to make loud noise. I figured it was the fan. With an aid of this YouTube movie (see below) I was able to replace the fan and repair it. Thanks luisfilipe1983!

Assembling the tools

Opening the back

Removing the keboard

Going inside

Getting closer...

Deep inside

Fixed!


The YouTube Movie:

Wednesday, October 1, 2014

Starting a PostDoc At UC Berkeley

I'm starting A postdoc fellowship at UC Berkeley Seismological Laboratory (BSL) under the supervision of Prof. Richard Allen. For the next year or two I'll be busy working on the Earthquake Early Warning System (EEWS) algorithms. My first project will be to figure how to implement BSL EEWS algorithms for the Israeli Seismological Network. California here we come...

Tuesday, December 31, 2013

Starting to work at GII

That's it. I've finished my  2 years Post-Doc adventure at TAU.
I'm now starting to work on the Israel national Earthquake Early Warning System (EEWS).
As of tomorrow, I'm going to work at the Seismology Division of the Geophysical Institute of Israel (GII).

Tuesday, November 26, 2013

Real-Time Kernel for RaspberryPi (Take II)

After unsuccessful attempt to compile Archlinux-rt kernel,
I encountered a very good guide describing how to compile a Debian Linux-rt kernel with Xenomai for the RaspberryPi. and a completing page for creating the RPI image.
I can't say I understand it completely, but I managed to combine to two guides and compile my own RaspberryPi Linux-rt kernel.

I'm sure some of the steps are missing and things change very quickly (e.g. links, kernel versions and patches) so anyone following my steps should take that into consideration and adapt to changes.

I started by installing a fresh Debian 32bit x86 on a virtual box.
Follow the picnc guides but remember to adapt the patches and kernel version differences.

I'll try to redo the process and give here a complete guide at a later stage.

Thursday, August 29, 2013

EQMet TSA-SMA setup for SeisComp3

The following post will describe how to install EQMet INTERNET-READY STRONG MOTION ACCELEROGRAPH (TSA-SMA) and set it up for SeisComp3.


Following the instruction in the manual:
The initial steps are to connect the TSA-SMA to electricity, GPS and a computer via the USB device connection.
Then, connect to the device via serial connection.
On windows: read the manual and follow the orders using putty. Don’t forget to download the driver for the device from EQMet site (http://www.eqmet.com/eng/Linux-cdc-acm.zip)

On Linux:
install cu:
# sudo yum install uucp
get the tty of the connection:
# dmesg | grep 'tty'
on my system it was: ttyACM0 but it can be ttyS[1234] or serial.
next I connect to the device:
# cu -l /dev/ttyACM0 -s 115200
it takes a while to prompt for user and password (root,kmi) press enter if does not prompt.
I need to assign an IP based on the mac address so I check the mac:
# ifconfig
after I got the IP, it is easy to connect via http sftp and ssh.

for static IP, edit /etc/network/interfaces:
replace:
iface eth0 inet dhcp
with:
iface eth0 inet static
address <IP>
netmask <MASK>
gateway <GATEWAY>
dns-nameservers <DNS-SERVER>

connect to the device via http.
Next go to Parameters:hardware menu option.
update to your preferable network-station-location-channel ID in:
- Network ID
- Unit ID and Site ID
- dig1 ChX ID
- dig1 ChX Location code
where X in channel 1, 2 or 3.

press OK.

Next, following the response mail I got for EQmet support regarding seedlink connection:
Sir,

The limited version of Rockhound in the TSA-SMA does not support continuous recording. However, you can approximate continuous recording by adding the SEEDLink mirror module and configuring it to accept data from the RockToSlink module at address 127.0.0.1, port 18000 (or as configured). You should add this module after installing and adding in the RockToSlink module.

This module writes the received MiniSEED packets out to storage and breaks the files at approximately the specified duration (depending on MiniSEED packet contents). If you add this module, you should also consider adding an auto file delete module to avoid filling the storage. A file sender module may also be useful (SSH, FTP, or email).

The RockToSlink module for Rockhound is provided by ISTI, New York. The module and documentation is available from ISTI at http://www.isti2.com/RockToSLink/

Technical support for RockToSLink is available from ISTI at isti-info@isti.com. SEEDlink is a public domain program and protocols. Support for these programs including detailed protocol descriptions or installation and setup of the programs is not provided by Kinemetrics.

Regards,
EQMet, Technical Support
support@eqmet.com
installe the RockToSlink module and add it to the workflow:

1. Download the rocktoslik module from http://www.isti2.com/RockToSLink/dist_pub/:
> wget -v http://www.isti2.com/RockToSLink/dist_pub/RockToSLink_v1.0_dist.zip
2. Unzip it:
> unzip RockToSLink_v1.0_dist.zip
following the instractions in http://www.isti2.com/RockToSLink/current_dist/docs/RockToSLink_ovr.html:
3. sftp to the device:
> sftp root@<IP>
4. Change directory to module location:
> cd  /usr/rock/SMARTSDist/injar
5 Upload the module-update jar file:
> put RockToSLink_update.jar
6. connect via ssh:
> ssh root@<IP>
7. Change directory to module location:
> cd  /usr/rock/SMARTSDist/injar
8. Rename "RockToSLink_update.jar" to "update.jar"
> mv RockToSLink_update.jar update.jar
9. Reboot the instrument:
> reboot
Now add the RockToLink module to device layout:
10. In the device http menue Parameters: Advanced features, add options for Add/Remove Modules, Replace Modules and Advanced Modules.
11. In Module Add, add the RockToLink module.
12. Press apply changes now.

Now go to system operation and make sure all settings are correct.

enjoy.


Thursday, August 1, 2013

Machinoid Real-Time Linux for RaspberryPi

Download and install Machinoid following: http://www.machinoid.com/?p=10
apt-get install libtool gfortran
apt-get install libusb-1.0
install xenomai following the instructions in the link above.
sysctl vm.min_free_kbytes=8192
vi /etc/ld.so.conf.d/xenomai.conf
add 2 lines:
  # xenomai default configuration
  /usr/xenomai/lib
ldconfig
install wifi drivers (after I got problems with rt2800usb driver):
apt-get install firmware-ralink

create new file /etc/modprobe.d/8192cu.conf
add:
# Disable power saving
options 8192cu rtw_power_mgnt=0 rtw_enusbss=1 rtw_ips_mode=1

in /etc/dhcp/dhcp.conf
uncomment the timeout line and edit:
timeout 100;


Sunday, July 7, 2013

LSM303 accelerometer on RaspberryPi

After getting my new LSM303 accelerometer, I got my great electronics tec. Daviv Shtibelman to wire it up for RPI.
Connecting LSM303 pins 1 (SCL), 2 (SDA), 6 (GND) and 8 (V3V) to pins 5 (SCL), 3 (SDA), 6 (GND) and 1 (V3V) on the Pi board.
On my ArchlinuxARM OS I had to install i2c-tools:
> pacman -S i2c-tools
And add i2c modules:
> echo i2c-dev > /etc/modules-load.d/ i2c.conf
> echo i2c-bcm2708 >> /etc/modules-load.d/ i2c.conf
> reboot

now to check it works:
> i2cdetect -y 1
got a message like this:
     0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f
00:          -- -- -- -- -- -- -- -- -- -- -- -- --
10: -- -- -- -- -- -- -- -- -- 19 -- -- -- -- 1e --
20: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
30: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
40: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
50: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
60: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
70: -- -- -- -- -- -- -- --

Now, following this, I test it and it works. Now I'm up for creating my own code.

Made some changes to the code:

main.cpp
#include"LSM303DLHC.h"
#include<stdio.h>
#include <unistd.h>


/*
   getHeading() calculates a tilt-compensated heading.
   A float between 0 and 360 degrees is returned. You need
   to pass this function both a magneto and acceleration array.
 
   Headings are calculated as specified in AN3192:
   http://www.sparkfun.com/datasheets/Sensors/Magneto/Tilt%20Compensated%20Compass.pdf
 
*/

float getHeading(LSM303& lsm303dlhc);

#define PI 3.141592654


int main(void)
{
    uint8_t bajt;
    const char *fileN = "/dev/i2c-1";
    LSM303 lsm303dlhc(fileN);

    lsm303dlhc.enalbe();

    while(1)
    {
        lsm303dlhc.readAccelerationRaw();
        lsm303dlhc.readMagnetometerRaw();
        printf("acc [m/s^2]: \e[27;1;31m %f \e[m \e[27;1;32m %f \e[m \e[27;1;34m %f \e[m mag:  \e[27;1;31m %d \e[m  \e[27;1;32m %d \e[m \e[27;1;34m %d\e[m %fdeg\n",
               (int16_t)lsm303dlhc.acc_x_raw*0.00957,(int16_t)lsm303dlhc.acc_y_raw*0.00957,-(int16_t)lsm303dlhc.acc_z_raw*0.00957,
               (int16_t)lsm303dlhc.mag_x_raw, (int16_t)lsm303dlhc.mag_y_raw, (int16_t)lsm303dlhc.mag_z_raw,getHeading(lsm303dlhc));
        usleep(10);
       }

    }


float getHeading(LSM303& lsm303dlhc)
{
  float heading,pitch,roll,xh,yh,zh;
  // see section 1.2 in app note AN3192
  int magValue[3];
  float accelValue[3];
  magValue[0] = (int16_t)lsm303dlhc.mag_x_raw;
  magValue[1] = (int16_t)lsm303dlhc.mag_y_raw;
  magValue[2] = (int16_t)lsm303dlhc.mag_z_raw;
  accelValue[0] = (int16_t)lsm303dlhc.acc_x_raw*0.000976531;
  accelValue[1] = (int16_t)lsm303dlhc.acc_y_raw*0.000976531;
  accelValue[2] = -(int16_t)lsm303dlhc.acc_z_raw*0.000976531;

    // see appendix A in app note AN3192
  pitch = asin(-accelValue[0]);
  roll = asin(accelValue[1]/cos(pitch));

  xh = magValue[0] * cos(pitch) + magValue[2] * sin(pitch);
  yh = magValue[0] * sin(roll) * sin(pitch) + magValue[1] * cos(roll) - magValue[2] * sin(roll) * cos(pitch);
  zh = -magValue[0] * cos(roll) * sin(pitch) + magValue[1] * sin(roll) + magValue[2] * cos(roll) * cos(pitch);

  heading = 180*atan2(yh,xh)/PI;

  if (heading <0)
    heading += 360;

  return heading;
}

LSM303DLHC.cpp:
#include"LSM303DLHC.h"
#include<math.h>
#include<stdio.h>

/*Conection to Raspberry PI:
 LSM303     Raspberry PI
 VDD    ->  3V3(PIN 1)
 SDA    ->  SDA(PIN 3)
 SCL    ->  SCL(PIN 5)
 GND    ->  GND(PIN 6)
*/

#define LSM303DLHC_MAG_ADDRESS            (0x3C >> 1)
#define LSM303DLHC_ACC_ADDRESS            (0x32 >> 1)

LSM303::LSM303(const char * i2cDeviceName) : i2c_lsm303(i2cDeviceName)
{

}

uint8_t LSM303::readAccRegister(uint8_t regAddr)
{
    i2c_lsm303.addrSet(LSM303DLHC_ACC_ADDRESS);
    return i2c_lsm303.readByte(regAddr);
}

uint8_t LSM303::readMagRegister(uint8_t regAddr)
{
    i2c_lsm303.addrSet(LSM303DLHC_MAG_ADDRESS);
    return i2c_lsm303.readByte(regAddr);
}

void LSM303::writeAccRegister(uint8_t regAddr,uint8_t byte)
{
    i2c_lsm303.addrSet(LSM303DLHC_ACC_ADDRESS);
    i2c_lsm303.writeByte(regAddr, byte);

}

void LSM303::writeMagRegister(uint8_t regAddr, uint8_t byte)
{
    i2c_lsm303.addrSet(LSM303DLHC_MAG_ADDRESS);
    i2c_lsm303.writeByte(regAddr, byte);

}

void LSM303::enalbe(void)
{
   writeAccRegister(LSM303_CTRL_REG1, 0b10010111);
   writeAccRegister(LSM303_CTRL_REG4, 0b00001000);

   writeMagRegister(LSM303_MR_REG, 0x00);
}

void LSM303::readAccelerationRaw(void)
{
    uint8_t block[6];

    i2c_lsm303.addrSet(LSM303DLHC_ACC_ADDRESS);

    i2c_lsm303.readBlock(0x80 | LSM303_OUT_X_L_A, sizeof(block), block);
    acc_x_raw = (int16_t)(block[0] | (block[1] << 8)) >> 4;
    acc_y_raw = (int16_t)(block[2] | block[3] << 8) >> 4;
    acc_z_raw = (int16_t)(block[4] | block[5] << 8) >> 4;

}

void LSM303::readMagnetometerRaw(void)
{
    uint8_t block[6];

    i2c_lsm303.addrSet(LSM303DLHC_MAG_ADDRESS);
    i2c_lsm303.readBlock(0x80 | LSM303_OUT_X_H_M, sizeof(block), block);

    mag_x_raw = (int16_t)(block[1] | block[0] << 8);
    mag_y_raw = (int16_t)(block[5] | block[4] << 8);
    mag_z_raw = (int16_t)(block[3] | block[2] << 8);

}

void LSM303::readAcceleration(void)
{
    readAccelerationRaw();
}

Sunday, June 30, 2013

Real-Time Kernel for RaspberryPi

After installing and running ArchLinuxARM on my RaspberryPi, I tried to compile a real-time kernel.
With the help of moonman, following this ArchLinuxARM forum thread, I downloaded the RT patch:
https://www.kernel.org/pub/linux/kernel/projects/rt/3.6/patch-3.6.11.4-rt36.patch.gz
and the PKGBUILD:
# git clone https://github.com/archlinuxarm/PKGBUILDs.git
following this, I updated pacman:
# pacman -Syy
and installed some packages:
# pacman -S kernel26-headers file base-devel abs
next
# cd PKGBUILDs/core/linux-raspberrypi
and edit the PKGBUILD file to include the patch in the source array:

source=('config'
        'change-default-console-loglevel.patch'
        'usb-add-reset-resume-quirk-for-several-webcams.patch'
        'args-uncompressed.txt'
        'boot-uncompressed.txt'
        'imagetool-uncompressed.py'
        'https://raspy-juice.googlecode.com/svn/trunk/linux-rtc/0001-rtc-pcf8523.patch'
        'https://raspy-juice.googlecode.com/svn/trunk/linux-rtc/0002-pcf8523-i2c-register-dt.patch'
        'patch-3.6.11.4-rt36.patch')

add a patch line:
patch -Np1 -i "${srcdir}/patch-3.6.11.4-rt36.patch"

and add md5sums at the end of the PKGBUILD file. to get the correct number run:
# md5sum patch-3.6.11.4-rt36.patch

next for the build:
# makepkg --asroot -Acs

The process has failed with some rejections of patches. So we need to manually apply the patches:
first look for rejections:
# find . -name "*.rej"
then edit the file and the .rej file:
# vim XXX.c XXX.c.rej
replacing XXX with the file path and name.
Then use :split to split the screen :next to change one screen to .rej file and ctrl+ww to switch between files.
add the lines marked by + and remove lines marked by - (lines with no +- sign are used for orientation.

after manually patching all files, edit the PKGBUILD file to ignore the kernel git and the patches (adding # at the start of the line)
add a line to src/linux/security/apparmor/sid.c:
"#include <linux/cache.h>"

re-run:
# makepkg --asroot

Now for the installation part:
# pacman -U linux-headers-raspberrypi-3.6.11-11-armv6h.pkg.tar.xz linux-raspberrypi-3.6.11-11-armv6h.pkg.tar.xz

And reboot.

Well, it didn't work.

Saturday, June 1, 2013

RaspberryPi initial steps

I initiated a new project involving low-cost micro-computer RaspberryPi.
In order to begin I ordered some basic parts (costs are in USD for Israeli market, should be cheaper elsewhere):
a RaspberryPi (55$)
5v charger with microusb adapter (10$)
8GB SD card (12$)
HDMI cable (8$)

and some extra parts:
3.5" LCD screen (45$)
micro-keyboard (62$)
micro-sd low profile adapter (11.9$)
wifi dongle (18.6$)

Not so cheap but might be worth it.

after I got a couple of RaspberryPi (model B rev2 512MB RAM) devices for a project I'm working on (details about the project will come later if it will work).
I bought a SD 8GB card and hooked it to the my desktop computer.
I downloaded the ARCLinux image unzipped it and followed the instructions:
Replacing sdX with the location of the SD card, run:
dd bs=1M if=/path/to/archlinux-hf-2013-06-15.img of=/dev/sdX
In order to increase the SD space to it's full 8GB I used gparted.

upgrade system:
> pacman -Sy pacman
> pacman-key --init
> pacman -Syu
sync (3 times):
# sync
reboot:
# reboot
Now I can add packages using pacman. to look for a package use:
> pacman -Ss <package name>
then select the one you need and install by:
> pacman -S <package name>

I am currently testing the Pi with the screen and 8 AA batteries to see how long it can work. I plan to build a case with a built-in 3.5" screen and a miniature keyboard so I can have a control unit for other RaspberryPi devices. My initial thoughts of using an old Gamboy case are now switched (with the encouragement of my electronic tec. guy David Shtibelman) to build a costume made case. Of course, David is the one to build it, since I have no skills to do that.



Wednesday, January 16, 2013

Fedora 17 Linux packges

These are MY needed packages, feel free to follow my steps or ignore this.
1. Add rpmfusion to yum repo list:
 > sudo yum localinstall --nogpgcheck http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-stable.noarch.rpm http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-stable.noarch.rpm
2.
sudo yum install -y yumex tcsh gmt ImageMagick ipython scipy scitools spyder gthumb nautilus-open-terminal bluecurve-icon-theme gnome-applets gnome-themes bluecurve-gnome-theme filezilla gimp swig lshw lshw-gui gnome-tweak-tool libreoffice gv gcc gcc-gfortran gcc-c++ dconf-editor tigervnc-server
install chrome from google
install virtualbox from oracle site.
copy panels:
on old machine: dconf dump /org/gnome/gnome-panel/ > panels
on new machine: dconf load /org/gnome/gnome-panel/ < panels
change with dconf-editor:
/org/gnome/desktop/interface/icon-theme to Bluecurve
/org/gnome/desktop/wm/preferences/theme Clearlooks
/org/gnome/desktop/background/show-desktop-icons True
change user shell to tcsh: sudo usermod -s /bin/tcsh USERNAME
install microsoft fonts:
wget http://www.my-guides.net/en/images/stories/fedora12/msttcore-fonts-2.0-3.noarch.rpm
sudo rpm -ivh msttcore-fonts-2.0-3.noarch.rpm

6.Install obspy as root:
sudo easy_install -N obspy.core obspy.mseed obspy.sac obspy.gse2 obspy.imaging obspy.signal obspy.arclink obspy.datamark obspy.db obspy.earthworm obspy.fissures obspy.iris obspy.neries obspy.realtime obspy.segy obspy.seg2 obspy.seisan obspy.seishub obspy.sh obspy.taup obspy.wav obspy.xseed

Tuesday, January 1, 2013

Done with the PhD., Moving forward

Finally, I finished my PhD. and now it time to move forward.
Taking my Professors advice, I'm making the first steps in the world of seismology and I'm going to spend the next few years in Tel-Aviv University, working with Dr. Alon Ziv and Dr. Hillel Wust-Bloch.
The main work is building and maintain a local mini seismic-array network. I am going to actually place the seismometers, connect the dataloggers and build the server to collect the data. We are going to use different devices - Lennartz LE-3Dlite1DV MkII seismometers, Trillium broadband seismometers, TSA-SMA accelerometers and Summit M Hydra dataloggers. On the server side, We are going to use Seiscomp3 as our main framework, but we probably research for new algorithms and Seiscomp3 modules to better suit it to our needs.
Me getting the PhD. diploma from Prof. Rivka Carmi, presedent of BGU

Sunday, December 16, 2012

JoyWarrior24F14 Udev rules

The JW24F14 device is accessible as a HID only as root. In order to access it as a user, I had to write a UDEV rule with the help of this post. more info about udev rules can be found here.
issue the command:
> sudo vi /etc/udev/rules.d/10-JoyWarrior24F14.rules
and when in editing mode (i) add:

# Rules for accessing JoyWarrior24F14 HID as user:
SUBSYSTEM=="usb", ATTRS{idVendor}=="07C0", ATTRS{idProduct}=="0140", MODE:="666", GROUP="users"
SUBSYSTEM=="usb", ATTRS{idVendor}=="07c0", ATTRS{idProduct}=="1116", MODE:="666", GROUP="users"

Thursday, October 25, 2012

Python Event Listener - multiprocessing deamon

I wanted to have a file change listener such that if new data is added to a file I could run a code.
Following this post, I've written a modification so now the listener is working in multiprocessing.



#!/usr/bin/python
# A code to stream file as it grows
# by Ran Novitsky Nof Oct 25, 2012

import os,time
from multiprocessing import Process,Pipe

# change this file name
FileName = 'stream.txt'

# adapted from http://www.valuedlessons.com/2008/04/events-in-python.html


# the Event class take care of adding, removing and executing the functions
class Event:
    def __init__(self):
        self.handlers = set()

    def handle(self, handler):
        self.handlers.add(handler)
        return self

    def unhandle(self, handler):
        try:
            self.handlers.remove(handler)
        except:
            raise ValueError("Handler is not handling this event, so cannot unhandle it.")
        return self

    def fire(self, *args, **kargs):
        for handler in self.handlers:
            handler(*args, **kargs)

    def getHandlerCount(self):
        return len(self.handlers)

    __iadd__ = handle
    __isub__ = unhandle
    __call__ = fire
    __len__  = getHandlerCount
 
# the stream class takes care of the
# function to fire in case of an event
class Stream:
  def __init__(self,fname):
    self.fname = fname
    self.fd = open(fname,'r')
    self.get_stream()
  def get_stream(self):
    # this is the actual function to fire
    print self.fd.read(),
  def restart(self):
    self.fd.seek(0)
 
# the MockFileWatcher creats the multiprocessing deamon to alert on file change
class MockFileWatcher:
    def __init__(self,source_path,sleeptime=0.1):
        self.fileChanged = Event()
        self.source_path = source_path
        self.running=False
        self.deamon=False
        self.fd = None
        self.sleep = sleeptime
        # the next two lines can be set outside the class
        self.stream = Stream(source_path)
        self.fileChanged += self.stream.get_stream # add event handler
    def watchFiles(self):
        # open the file in non-blocking read mode
        self.fd = os.open(self.source_path,os.O_RDONLY | os.O_NONBLOCK)
        # go to the end
        os.lseek(self.fd,0,2)
        self.running=True
        self.deamon=False
        while self.running:
          # try to read
          if os.read(self.fd,1)!='':
            # New data was added
            os.lseek(self.fd,0,2) # go to the end of file
            self.fileChanged() # fire the event
          # see if the parent process has a message
          if self.chiled_conn.poll():
            # execute the message
            eval(self.chiled_conn.recv())
          # sleep for a while
          time.sleep(self.sleep)
    def watchFilesDeamon(self):
        # start the deamon
        self.parent_conn,self.chiled_conn = Pipe()
        p = Process(target=self.watchFiles)
        p.start()
        self.p = p  
        self.deamon=True
        return p
    def stop(self):
        # stop deamon/process
        if self.deamon:
          self.parent_conn.send('self.stop()')
          print "Hold while stopping child process..."
          # hold depends on the sleeping time
          if self.parent_conn.recv():
            pass
            #print "chiled is stopped"
          self.p.terminate() # terminate multiprocessing
          self.deamon=False
        # stop process
        self.running=False
        self.fd=None
        # if this is the process make sure parent know we are done
        if not self.deamon:  self.chiled_conn.send(True)      
        return True

if __name__=='__main__':
  watcher = MockFileWatcher(FileName)
  p = watcher.watchFilesDeamon()
  print 'from this point the listner is running at the background'
  print 'add more code here so you can see how it works.'

Try this by changing the file name at the top and run it. then add new lines to the file.

Wednesday, October 24, 2012

SeisComp3 - get events data

After installing  Seiscomp3 and running it for a while, now it is time to play with the data.
As a first step I want to be able to see the events outside the SeisComp3 environment.
This will later serve me when exporting the data to a web page or reports.
The steps described here are: 1) get a list of events, 2) export the data to an xml file and 3) print the data as a bulletin.
1) Get a list of events:
scevtls -d mysql://sysop:sysop@localhost/seiscomp3
Will give a list of event codes like:

Wrong 'begin' format -> setting to None
Setting start to 1970-01-01 00:00:00
Setting end to 2012-10-24 08:17:13
gfz2012uizn
gfz2012atlf
...
where gfz will be the agency prefix and the last 4 letters will be random code.
You can also set begin and end time to get only a time window events. see scevtls documentation.

2) For this example I select one event (in csh environment):
set EVENT = gfz2012uizn
Now we can export the event data to an xml file:
scxmldump -d mysql://sysop:sysop@localhost/seiscomp3 -E $EVENT -P -M -A -o test.xml
see scxmldump for more information

3) Print the data to stdout:
scbulletin -i test.xml
Note that it is possible to skip the xmp part by using:
scbulletin -d mysql://sysop:sysop@localhost/seiscomp3 -E $EVENT
also adding -3 as a flag will output even more details.
see scbulletin for more info.

I'll update later with a python script to export the data to a Google Earth kml file.

Monday, September 24, 2012

Installing SeisComp3 on fedora 17

After installing Fedora 17  I will now describe how to install MySQL server and SeisComp3.

first, install MySQL (following this):
> sudo yum install -y mysql mysql-server

enable and start MySQL service:
> sudo systemctl enable mysqld.service
> sudo systemctl start mysqld.service

Next, secure the server:
> sudo /usr/bin/mysql_secure_installation
(answer yes to all questions and change the MySQL root password)
If you need to change the database location from the default follow this (source):
Stop the service:
> systemctl stop mysqld.service
Copy from default to new location:
> cp -R -p /var/lib/mysql /path/to/new/datadir
Clean unneeded files:
> rm /path/to/new/datadir
(don't worry about error messages - the undeleted files are the databases)
Edit the configuration file:
>  gedit /etc/mysql/my.cnf
(change keyword value of datadir to the new location)
Restart the service:
> systemctl restart mysqld.service

Now to the SeisComp3 install (Zurich version for fedora16 modified for fedora 17):
Add needed packages:
> sudo yum install -y boost-filesystem boost-iostreams boost-program-options boost-serialization boost-signals
boost-thread boost-wave boost-regex boost-devel ncurses python-numeric numpy alsa-utils festival
Link installed libraries (so.1.48) to the needed packages (so.1.47):

> sudo ln -s /usr/lib64/libboost_iostreams.so.1.48.0 /usr/lib64/libboost_iostreams.so.1.47.0

> sudo ln -s /usr/lib64/libboost_filesystem.so.1.48.0 /usr/lib64/libboost_filesystem.so.1.47.0
> sudo ln -s /usr/lib64/libboost_system.so.1.48.0 /usr/lib64/libboost_system.so.1.47.0
> sudo ln -s /usr/lib64/libboost_regex.so.1.48.0 /usr/lib64/libboost_regex.so.1.47.0
> sudo ln -s /usr/lib64/libboost_thread-mt.so.1.48.0 /usr/lib64/libboost_thread-mt.so.1.47.0
> sudo ln -s /usr/lib64/libboost_program_options.so.1.48.0 /usr/lib64/libboost_program_options.so.1.47.0
> sudo ln -s /usr/lib64/libboost_signals.so.1.48.0 /usr/lib64/libboost_signals.so.1.47.0
Put key files in ~/.seiscomp3/key/ directory
Install using the setup and seiscomp config procedures described in the SeisComp3 documentations
add a source to env.sh file in your .bashrc:
source ~/seiscomp3/lib/env.sh

For the seattle release use:
>sudo yum install -y boost boost-devel python-numeric


Don't forget to download and extract to the seiscomp directory, the correct fedora seiscomp package (e.g. seiscomp3-zurich-xxxx.xxx.xx-fedora16-x86_64.tar.gz), the maps (e.g. seiscomp3-maps.tar.gz) and the methadata of GE network (e.g. seiscomp3-metadata-GE-2010.tar.gz).

Tuesday, August 28, 2012

Howto write to stdout XY

how to use python to write to a certain location (x,y) on the stdout (terminal):
copied form here
import sys
def print_there(x, y, text):
     sys.stdout.write("\x1b7\x1b[%d;%df%s\x1b8" % (x, y, text))
     sys.stdout.flush()