Sunday, September 8, 2019

Setting up ElasticSearch on EC2 - with remote connection configuration

September 08, 2019 by 
Install Java 8
sudo yum install java-1.8.0

Remove
sudo yum remove java-1.7.0-openjdk   

Get ES
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.4.rpm

Use the root user
sudo su

Install ES
sudo rpm --install elasticsearch-6.2.4.rpm     

Run ES on system startup
sudo chkconfig --add elasticsearch   

Go inside the elasticsearch directory
cd /usr/share/elasticsearch/

Install the plugins
sudo bin/elasticsearch-plugin install discovery-ec2    sudo bin/elasticsearch-plugin install repository-s3

Change the jvm heap size
sudo nano /etc/elasticsearch/jvm.options
->
-Xms512m
-Xmx512m

Change the configuration to allow remote access
nano /etc/elasticsearch/elasticsearch.yml
Add this
http.host: 0.0.0.0

Run 
sudo /etc/init.d/elasticsearch restart

or 

sudo service elasticsearch start


Then finally, check

Tuesday, August 6, 2019

Working With Date Time - In Depth

August 06, 2019 by 
A date in Python is not a data type of its own, but we can import a module named datetime to work with dates as date objects.

Example


import datetime
x = datetime.datetime.today()
x #> datetime.datetime(2019, 8, 6, 22, 39, 30, 864393)

The output is in the following order: ‘year’, ‘month’, ‘date’, ‘hour’, ‘minute’, ‘seconds’, ‘microseconds’

Parsing a string to datetime

my_date_time = datetime.datetime.strptime('8/3/19', '%m/%d/%y')
my_date_time #> datetime.datetime(2019, 8, 3, 0, 0)

Parsing any string format to datetime

from dateutil.parser import parse
parse('94, December 26, 2010, 10:51pm') #> datetime.datetime(1994, 12, 26, 22, 51)

Formatting datetime

my_date_time = datetime.datetime.strptime('8/3/19', '%m/%d/%y')
my_date_time.strftime('%m/%d/%y') #> '08/03/19'

Adjusting datetime

my_date_time = datetime.datetime.strptime('8/3/19', '%m/%d/%y')
my_date_time - datetime.timedelta(days=2) #> datetime.datetime(2019, 8, 1, 0, 0)

Syntax: datetime.timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)

Useful datetime functions

# create a datatime obj
dt = datetime.datetime(2019, 2, 15)

# 1. Get the current day of the month
dt.day #> 31

# 2. Get the current day of the week
dt.isoweekday() #> 5 --> Friday

# 3. Get the current month of the year 
dt.month  #> 2 --> February

# 4. Get the Year
dt.year  #> 2019

Get the last day of a month for any given date

import datetime
dt = datetime.date(1952, 2, 12)


import calendar
calendar.monthrange(dt.year,dt.month)[1] #> 29


Pandas date_range


import pandas as pd
import datetime
 
date1 = pd.Series(pd.date_range('2018-1-1 12:00:00', periods=7, freq='M'))
df = pd.DataFrame(dict(date_given=date1))
df



Sunday, August 4, 2019

Get list from pandas DataFrame column headers

August 04, 2019 by 
The groupby function can be used to concatenate data from multiple rows into one field.


Create a Dataframe 

import pandas as pd
import numpy as np

#Create a Dictionary of series
d = {'Name': pd.Series(['Alisa','Bobby','Cathrine','Madonna','Rocky','Sebastian','Jaqluine',
   'Rahul','David','Andrew','Ajay','Teresa']),
   'Age': pd.Series([26,27,25,24,31,27,25,33,42,32,51,47]),
   'Score': pd.Series([89,87,67,55,47,72,76,79,44,92,99,69])}
 
#Create a DataFrame
df = pd.DataFrame(d)
df

the resultant dataframe will be




Now lets get the values as a list by doing: 


column_names = df.columns.values.tolist()
column_names

the result will be


Thursday, August 1, 2019

Concatenating Rows in Python Pandas

August 01, 2019 by 
The groupby function can be used to concatenate data from multiple rows into one field.


Create a Dataframe 

import pandas as pd
import numpy as np

#Create a Dictionary of series
d = {'Name': pd.Series(['Alisa','Bobby','Cathrine','Madonna','Rocky','Sebastian','Jaqluine',
   'Rahul','David','Andrew','Ajay','Teresa']),
   'Age': pd.Series([26,27,25,24,31,27,25,33,42,32,51,47]),
   'Score': pd.Series([89,87,67,55,47,72,76,79,44,92,99,69])}
 
#Create a DataFrame
df = pd.DataFrame(d)
df

the resultant dataframe will be




Now lets group by age of the student name 


final_df = df.groupby('Age')['Name'].apply(', '.join)

final_df

the result will be


Monday, July 22, 2019

How to split a list inside a Dataframe cell into rows in Pandas

July 22, 2019 by  
       
temp = {'name' : ['Edmond', 'ALex'], 'cat' : [['Horror', 'Vengeance', 'Justice'], ['Romance', 'Sacrifice']]}


      
df = pd.DataFrame(temp)


      
df.cat.apply(pd.Series)


      
df.cat.apply(pd.Series) \
.merge(df, left_index = True, right_index = True)


df.cat.apply(pd.Series) \
.merge(df, left_index = True, right_index = True) \
.drop(['cat'], axis = 1)


df.cat.apply(pd.Series) \
.merge(df, left_index = True, right_index = True) \
.drop(['cat'], axis = 1) \
.melt(id_vars = ['name'], value_name = "cat") \
.drop(['variable'], axis= 1)

Amount to Words Convert via PostgreSQL

July 22, 2019 by 
asd



CREATE OR REPLACE FUNCTION dateToWords(the_date TIMESTAMP) RETURNS text AS $$
DECLARE
_month INTEGER;
_day  INTEGER;
_year INTEGER;
_hour INTEGER;
_minute INTEGER;
_year_text TEXT;
_hour_text TEXT;
_day_text TEXT;
_minute_text TEXT;
_exact_time TEXT;
_month_text TEXT;
e TEXT;
BEGIN
_month = EXTRACT(MONTH FROM the_date);
_day = EXTRACT(DAY FROM the_date);
_year = EXTRACT(YEAR FROM the_date);
_hour = EXTRACT(HOUR FROM the_date);
_minute = EXTRACT(MINUTE FROM the_date);


WITH Below20(Word, Id) AS
(
VALUES
  ('Zero', 0), ('One', 1),( 'Two', 2 ), ( 'Three', 3),
  ( 'Four', 4 ), ( 'Five', 5 ), ( 'Six', 6 ), ( 'Seven', 7 ),
  ( 'Eight', 8), ( 'Nine', 9), ( 'Ten', 10), ( 'Eleven', 11 ),
  ( 'Twelve', 12 ), ( 'Thirteen', 13 ), ( 'Fourteen', 14),
  ( 'Fifteen', 15 ), ('Sixteen', 16 ), ( 'Seventeen', 17),
  ('Eighteen', 18 ), ( 'Nineteen', 19 )
),
Below100(Word, Id) AS
(
  VALUES
   ('Twenty', 2), ('Thirty', 3),('Forty', 4), ('Fifty', 5),
   ('Sixty', 6), ('Seventy', 7), ('Eighty', 8), ('Ninety', 9)
)SELECT
CASE
  WHEN (_year % 1000) BETWEEN 1 AND 19 THEN
  CONCAT(
      (Select Word FROM Below20 WHERE ID=_year / 1000), ' Thousand ',
      (Select Word FROM Below20 WHERE ID=_year % 1000)
    )
  WHEN (_year % 1000) BETWEEN 20 AND 99 THEN
  CONCAT(
      (Select Word FROM Below20 WHERE ID=_year / 1000), ' Thousand ',
      (Select Word FROM Below20 WHERE ID=(_year % 1000) / 10), ' ',
      (Select Word FROM Below20 WHERE ID=(_year % 1000) / 10)
    )
  WHEN (_year % 1000) BETWEEN 100 AND 999 THEN
    CASE
        WHEN (_year % 1000) % 100 BETWEEN 1 AND 19 THEN
        CONCAT(
            (Select Word FROM Below20 WHERE ID=_year / 1000), ' Thousand ',
            (Select Word FROM Below20 WHERE ID=(_year % 1000) / 100), ' Hundred ',
            (Select Word FROM Below20 WHERE ID=((_year % 1000) % 100))
          )
        WHEN (_year % 1000) % 100 BETWEEN 20 AND 99 THEN
        CONCAT(
            (Select Word FROM Below20 WHERE ID=_year / 1000), ' Thousand ',
            (Select Word FROM Below20 WHERE ID=(_year % 1000) / 100), ' Hundred ',
            (Select Word FROM Below100 WHERE ID=((_year % 1000) % 100) / 10), ' ',
            (Select Word FROM Below20 WHERE ID=((_year % 1000) % 100) % 10)
          )
    END
END INTO _year_text;


WITH Below20(Word, Id) AS
(
 VALUES
   ('', 0), ('One', 1),( 'Two', 2 ), ( 'Three', 3),
   ( 'Four', 4 ), ( 'Five', 5 ), ( 'Six', 6 ), ( 'Seven', 7 ),
   ( 'Eight', 8), ( 'Nine', 9), ( 'Ten', 10), ( 'Eleven', 11 ),
   ( 'Twelve', 12 )
)
Select
    CASE
      WHEN _hour > 12 THEN (Select Word FROM Below20 WHERE ID= _hour - 12)
      ELSE (Select Word FROM Below20 WHERE ID= _hour)
END INTO _hour_text;



WITH Below20(Word, Id) AS
(
VALUES
  ('', 0), ('First', 1),( 'Second', 2 ), ( 'Third', 3),
  ( 'Fourth', 4 ), ( 'Fifth', 5 ), ( 'Sixth', 6 ), ( 'Seventh', 7 ),
  ( 'Eighth', 8), ( 'Ninth', 9), ( 'Tenth', 10), ( 'Eleventh', 11 ),
  ( 'Twelfth', 12 ), ( 'Thirteenth', 13 ), ( 'Fourteenth', 14),
  ( 'Fifteenth', 15 ), ('Sixteenth', 16 ), ( 'Seventeenth', 17),
  ('Eighteenth', 18 ), ( 'Nineteenth', 19 )
),
Below100(Word, Id) AS
(
   VALUES
    ('Twenty', 2), ('Thirty', 3)
)
Select
CASE
  WHEN _day BETWEEN 1 AND 19 THEN (Select Word FROM Below20 WHERE ID=_day)
  ELSE
  CONCAT(
      (Select Word FROM Below100 WHERE ID= _day / 10), ' ',
      (Select Word FROM Below20 WHERE ID= _day % 10)
   )
END INTO _day_text;



WITH Below20(Word, Id) AS
(
VALUES
  ('', 0), ('One', 1),( 'Two', 2 ), ( 'Three', 3),
  ( 'Four', 4 ), ( 'Five', 5 ), ( 'Six', 6 ), ( 'Seven', 7 ),
  ( 'Eight', 8), ( 'Nine', 9), ( 'Ten', 10), ( 'Eleven', 11 ),
  ( 'Twelve', 12 ), ( 'Thirteen', 13 ), ( 'Fourteen', 14),
  ( 'Fifteen', 15 ), ('Sixteen', 16 ), ( 'Seventeen', 17),
  ('Eighteen', 18 ), ( 'Nineteen', 19 )
),
Below100(Word, Id) AS
(
   VALUES
    ('Twenty', 2), ('Thirty', 3),('Forty', 4), ('Fifty', 5),
    ('Sixty', 6)
)
Select
CASE
  WHEN _minute BETWEEN 1 AND 19 THEN (Select Word FROM Below20 WHERE ID=_day)
  ELSE
  CONCAT(
      (Select Word FROM Below100 WHERE ID= _minute / 10), ' ',
      (Select Word FROM Below20 WHERE ID= _minute % 10)
   )
END INTO _minute_text;



SELECT
CASE
WHEN _hour BETWEEN 0 AND 12 THEN 'Morning'
WHEN _hour BETWEEN 13 AND 17 THEN 'Afternoon'
ELSE 'Evening'
END INTO _exact_time;



WITH nameOfTheMonth(monthAsText, Id) AS
(
VALUES
  ('January', 1), ('February', 2),( 'MARCH', 3), ( 'April', 4),
  ( 'May', 5), ( 'June', 6), ( 'July', 7), ( 'August', 8),
  ( 'September', 9), ( 'October', 10), ( 'November', 11), ( 'December', 12)
)
select monthAsText FROM nameOfTheMonth WHERE ID=_month INTO _month_text;


SELECT CONCAT(
      _day_text, ' day of', ' ',
      _month_text,', ',
     _year_text,', ',
     _hour_text,' ',
     _minute_text, ' in the ',
  _exact_time)
    INTO e;


return e;
END; $$
LANGUAGE PLPGSQL;


select dateToWords('May 15, 2018 5:38 PM')
Subscribe to: Posts (Atom)