In [1]:
import pandas as pd
In [5]:
%pwd
Out[5]:
In [15]:
# downloaded survey data; now need to change working directory to retrieve it
%cd /Users/alisonfowler/Docs/Jupityr-notebooks/
In [16]:
# Note that pd.read_csv is used because we imported pandas as pd
pd.read_csv("data/surveys.csv")
Out[16]:
In [17]:
surveys_df = pd.read_csv("data/surveys.csv")
In [18]:
surveys_df
Out[18]:
In [19]:
surveys_df.head()
Out[19]:
In [20]:
type(surveys_df)
Out[20]:
In [21]:
surveys_df.dtypes # the formats of different types of data
Out[21]:
In [ ]:
# CHALLENGE : DATAFRAMES
# 1. surveys_df.columns
# 2. surveys_df.shape Take note of the output of shape - what format does it return the shape of the DataFrame in?
# HINT: More on tuples, here.
# 3. surveys_df.head() Also, what does surveys_df.head(15) do?
# 4. surveys_df.tail()
In [22]:
surveys_df.columns # returns column names
Out[22]:
In [23]:
surveys_df.shape # dimensions - returns the shape as a vector
Out[23]:
In [41]:
surveys_df.head() # displays first five rows of the dataframe
Out[41]:
In [24]:
surveys_df.head(15) # displays first fifteen rows of the dataframe
Out[24]:
In [25]:
surveys_df.tail() # displays last five rows
Out[25]:
In [26]:
pd.unique(surveys_df['species_id']) # list all the unique species
Out[26]:
In [ ]:
# CHALLENGE : STATISTICS
# 1. Create a list of unique site ID’s (“plot_id”) found in the surveys data. Call it site_names.
# How many unique sites are there in the data? How many unique species are in the data?
# 2. What is the difference between len(site_names) and surveys_df['plot_id'].nunique()?
In [27]:
site_names=pd.unique(surveys_df['plot_id']) # create a new object with site names
In [28]:
site_names # look at new object
Out[28]:
In [29]:
site_names.shape # how many sites are there
Out[29]:
In [30]:
species_names=pd.unique(surveys_df['species_id']) # create a new object with species names
In [31]:
species_names # look
Out[31]:
In [32]:
species_names.shape # how many species are there
Out[32]:
In [70]:
len(species_names)
Out[70]:
In [33]:
len(site_names) # length of vector
Out[33]:
In [34]:
surveys_df['plot_id'].nunique() # number of unique values in the vector
# different from previous line b/c the vector could have repeating values.
# in this case, it returns the same number since all values are unique
Out[34]:
In [35]:
surveys_df['weight'].describe() # get basic stats for a given column
Out[35]:
In [36]:
surveys_df['weight'].min() # extract a stat
Out[36]:
In [37]:
# Groups
# Group data by sex
grouped_data = surveys_df.groupby('sex')
In [38]:
# Summary statistics for all numeric columns by sex
grouped_data.describe()
Out[38]:
In [39]:
# Provide the mean for each numeric column by sex
grouped_data.mean()
Out[39]:
In [ ]:
# CHALLENGE : SUMMARY DATA
# 1. How many recorded individuals are female F and how many male M?
# 2. What happens when you group by two columns using the following syntax and then calculate mean values?
# grouped_data2 = surveys_df.groupby(['plot_id', 'sex'])
# grouped_data2.mean()
# 3. Summarize weight values for each site in your data.
# HINT: you can use the following syntax to only create summary statistics for one column in your data.
# by_site['weight'].describe()
In [42]:
grouped_data.count() # number of recorded individuals
# females: 15,690
# males: 17,348
# apparently some missing data for hindfoot length and weights?
Out[42]:
In [48]:
grouped_data2 = surveys_df.groupby(['plot_id', 'sex']) # create a new object grouped by two variables
In [49]:
grouped_data2.mean() # displays mean for each variable grouped by sexes within each plot/site
Out[49]:
In [52]:
grouped_data2['weight'].describe() # gives information about weights within groups
Out[52]:
In [53]:
# group by site only
grouped_data3 = surveys_df.groupby('plot_id')
In [59]:
grouped_data3.mean()
Out[59]:
In [60]:
grouped_data3['weight'].describe() # gives information about weights within sites
# but this output looks slightly different from the one on the tutorial site, which has the columns & rows switched
Out[60]:
In [61]:
# Quickly creating summary counts
# Count the number of samples by species
species_counts = surveys_df.groupby('species_id')['record_id'].count()
print(species_counts)
In [63]:
surveys_df.groupby('species_id')['record_id'].count()['DO'] # sample count for one particular species
Out[63]:
In [ ]:
# CHALLENGE : MAKE A LIST
# What’s another way to create a list of species and associated count of the records in the data?
# Hint: you can perform count, min, etc. functions on groupby DataFrames in the same way you can perform them
# on regular DataFrames.
In [66]:
surveys_df.head()
Out[66]:
In [77]:
grouped_data4 = surveys_df.groupby('species_id')
In [78]:
grouped_data4.count() # this works?
Out[78]:
In [79]:
# Basic Math functions
# Multiply all weight values by 2
surveys_df['weight']*2
Out[79]:
In [80]:
# Plotting
# Make sure figures appear inline in Ipython Notebook
%matplotlib inline
In [81]:
# Create a quick bar chart
species_counts.plot(kind='bar'); # what does the semi-colon do?
In [82]:
total_count = surveys_df.groupby('plot_id')['record_id'].nunique()
In [83]:
# Let's plot that too
total_count.plot(kind='bar');
In [84]:
# CHALLENGE: PLOTS
# 1. Create a plot of average weight across all species per site.
# 2. Create a plot of total males versus total females for the entire dataset.
In [85]:
site_weights = surveys_df.groupby('plot_id')['weight'] # group weights by site
In [86]:
site_weights.head()
Out[86]:
In [88]:
site_mean_weights = site_weights.mean() # extract means
In [89]:
site_mean_weights.head()
Out[89]:
In [92]:
site_mean_weights.plot(kind = 'bar'); # plot mean weights of sites
In [96]:
sex_counts = surveys_df.groupby('sex')['record_id'].count()
In [99]:
print(sex_counts)
In [98]:
sex_counts.plot(kind = 'bar'); # plot number of records of each sex of entire data set
In [ ]:
# SUMMARY PLOTTING CHALLENGE
# Create a stacked bar plot, with weight on the Y axis, and the stacked variable being sex.
# The plot should show total weight by sex for each site.
In [100]:
# Example data for making the plot
d = {'one' : pd.Series([1., 2., 3.], index=['a', 'b', 'c']), 'two' : pd.Series([1., 2., 3., 4.], index=['a', 'b', 'c', 'd'])}
In [101]:
pd.DataFrame(d)
Out[101]:
In [103]:
# Plot stacked data so columns 'one' and 'two' are stacked
my_df = pd.DataFrame(d)
my_df.plot(kind='bar', stacked=True, title="Example graph")
Out[103]:
In [104]:
# let's try to do this with weights by sites and sex
site_sex = surveys_df.groupby(['plot_id', 'sex'])
In [105]:
site_sex_count = site_sex['weight'].sum()
In [106]:
print(site_sex_count)
In [108]:
site_sex_count.unstack()
# transforms grouped data into columns
Out[108]:
In [111]:
site_sex_count.plot(kind='bar', stacked=True, title="Total weight by site and sex")
# so if we don't unstack first, it doesn't allow us to stack two variables within a site
Out[111]:
In [112]:
to_plot = site_sex_count.unstack()
In [113]:
to_plot.plot(kind='bar', stacked=True, title="Total weight by site and sex") # yay
Out[113]:
