Recomendation Systems

    
      import pandas as pd
      import numpy as np
      import matplotlib.pyplot as plt
      import matplotlib.cm as cm
      import matplotlib.patches as mpatches
      import matplotlib.cm as cm
      import plotly.express as px
      import seaborn as sns
      from ttictoc import tic,toc
      import networkx as nx
      import seaborn as sns
      import re as re
      from networkx.algorithms.community.centrality import girvan_newman
      from networkx.algorithms.centrality import edge_betweenness_centrality
      from networkx.algorithms.community.quality import modularity
      import community.community_louvain
      import operator
      from fa2l import force_atlas2_layout
      from wordcloud import WordCloud
      import warnings
      from collections import Counter
    
  

    
      import os
      from sklearn.preprocessing import StandardScaler
      from scipy.cluster.hierarchy import dendrogram
      from sklearn.cluster import AgglomerativeClustering
      from sklearn.cluster import SpectralClustering
    
  

    
    df = pd.read_csv("data/nutrition.csv", index_col = 0)
    df.head()
    
  

    
      def obtain_val(string):
      val = re.findall('\d*\.?\d+',string)
      return "".join(val)
      
      df.drop(['serving_size'], axis=1)
      #df['total_fat']=df['total_fat'].apply(lambda x: obtain_val(x)).astype(float)
      df2 = df.filter(['name'], axis=1)
      df = df.drop(columns=['name'])
      
      df = df.astype(str)
      
      #df = df.replace("",0)
      df = df.applymap(obtain_val)
      df = df.mask(df == '')
      df = df.astype(float)
      df2 = df2.join(df)
      
      df2.head()
    
  

    
      cols_to_keep = ["calories", "total_fat", "cholesterol", "sodium", "carbohydrate", "glucose", "sugars", "saturated_fatty_acids", "monounsaturated_fatty_acids", "polyunsaturated_fatty_acids", "fatty_acids_total_trans", "water"]
      df_clust = df2.loc[:,cols_to_keep]
      df_clust.head(10)
    
  

    
      fig = plt.figure(figsize=(13,10)).gca(projection = "3d")
      fig.scatter(df_clust["total_fat"], df_clust["calories"], df_clust["sugars"])
      fig.set_xlabel("Total Fat")
      fig.set_ylabel("Calories")
      fig.set_zlabel("Sugars")
      plt.show()
    
  

    
      fig, axs = plt.subplots(3,figsize=(13, 10))

      axs[0].scatter(x=df_clust["total_fat"], y=df_clust["sugars"], marker="o")
      axs[0].set_xlabel("Total Fat")
      axs[0].set_ylabel("Sugars")

      axs[1].scatter(x=df_clust["total_fat"], y=df_clust["calories"], marker="o")
      axs[1].set_xlabel("Total Fat")
      axs[1].set_ylabel("Calories")

      axs[2].scatter(x=df_clust["sugars"], y=df_clust["calories"], marker="o")
      axs[2].set_xlabel("Sugars")
      axs[2].set_ylabel("Calories")

      plt.show()
    
  

    
      scaler = StandardScaler()
      X = df_clust
      
      scaled_df = pd.DataFrame(scaler.fit_transform(X), columns = X.columns)
    
  

    
def dendogram_plot(md, **kwargs):
        count = np.zeros(md.children_.shape[0])
        n_samples = len(md.labels_)
        for (i, merge) in enumerate(md.children_):
            count_sum = 0
            for child_idx in merge:
                if child_idx < n_samples:
                    count_sum += 1
                else:
                    count_sum += count[child_idx - n_samples]
            count[i] = count_sum
        linkage_matrix = np.column_stack([md.children_,md.distances_,count]).astype(float)
        # Plot dendogram
        dendrogram(linkage_matrix, **kwargs)

      X = scaled_df

      # Model definition by using Agglomerative Clustering from sklearn. 
      # No references of the desired number of clusters.
      md = AgglomerativeClustering(distance_threshold=0, n_clusters=None)

      # Finally, let's create a dendogram by fitting our reduced dataframe.
      clustered_model = md.fit(X)
    
  

    
      # Representation of the fitted dendrogram
      fig = plt.figure(figsize=(12,10))
      plt.title("Dendogram - Depth: 3")
      dendogram_plot(clustered_model, truncate_mode ='level', p=3)
      plt.xlabel("Number of ingredients/elements per node")
      plt.savefig('Clustering_Dendogram.png')
      plt.show()
    
  

    
      # Set a specific number of clusters based on a distance_threshold
      data_fitting = AgglomerativeClustering(distance_threshold=90,n_clusters=None).fit(scaled_df)
      
      # The unique count for each cluster is:
      np.unique(data_fitting.labels_,return_counts=True)
      
      # Matches our expectations.
      (array([0, 1, 2, 3, 4, 5, 6]),
      array([1709, 3930,  357,   13,    8,   37, 2735]))
    
  

    
      # For later use, we will add the cluster label to each ingredient in the non-normalized dataframes.
      df_clust['Cluster num.'] = data_fitting.labels_.astype(str)
      df2['Cluster num.'] = data_fitting.labels_.astype(str)
      # Let's replot the previous scatter plots showing the our new clusters.
      sns.lmplot(x = 'total_fat', y='sugars', data=df_clust, hue='Cluster num.', fit_reg=False)
      plt.show()
    
  

    
      # Same between sugars and calories
      sns.lmplot(x = 'sugars', y='calories', data=df_clust, hue='Cluster num.', fit_reg=False)
      plt.show()
    
  

    
      # Finally, we will assign the mean saturated_fat value from each cluster to the ingredients with non-defined value (NaN)
      # NOTE: Pandas excludes NaN values when creating the mean.

      mean_satFat = df2.groupby('Cluster num.')['saturated_fat'].mean()
      mean_satFat

      Cluster num.
      0     4.362579
      1     4.089225
      2    18.600280
      3     4.815385
      4     2.725000
      5     2.311111
      6     0.767811
      Name: saturated_fat, dtype: float64
    
  

    
      df2["saturated_fat"] = np.where(pd.isna(df2["saturated_fat"]), mean_satFat[df2["Cluster num."]], df2["saturated_fat"])
      df2.head(10)
    
  

    
      # Drop cluster number column
      df = df2.drop(columns=["Cluster num."])
      df.head(5)
    
  

    
      df=pd.read_csv('data/cleaned_data.csv',index_col=0)
      df=df.set_index('name')

      # copy the data
      df_notnorm=df.copy()
      df_norm = df.copy()

      # apply normalization techniques
      for column in df_norm.columns:
      df_norm[column] = (df_norm[column] - df_norm[column].min()) / (df_norm[column].max() - df_norm[column].min())

      df_norm.head()
    
  

    
      #NetworkX Graph to store the network
      percentage=0.15
      criteria=int(percentage*8789)

      df=df_norm.copy()

      F= nx.Graph()
      for el in list(df):
          F.add_node(el,Role='Feature')
      for el in df.index:
          F.add_node(el,Role='Aliment')
          
      for f in df.columns:
          sorted_df=df.sort_values(by=[f],ascending=False).iloc[:criteria,0]
          for idx,el in enumerate(sorted_df.index):
              F.add_weighted_edges_from([(f,el,sorted_df[idx])])

      nodes_d0 = [node for node,degree in dict(F.degree()).items() if degree == 0]
      sF=F.copy()
      sF.remove_nodes_from(nodes_d0)

      print("There are " +str(F.number_of_edges())+" edges in our network.")
      print(len(nodes_d0))
      print('Number of nodes:',sF.number_of_nodes())
      print('NUmber of edges:',sF.number_of_edges())
    
  

    
      There are 8863 nodes in the network.
      There are 97532 edges in the network.
      143
      Number of nodes: 8720
      NUmber of edges: 97532
    
  

    
      color_map=[]
      degree=[]
      for node in F.nodes:
          if F.nodes[node]['Role']=='Feature':
              color_map.append('#157B87')
              degree.append(300)
          else:
              color_map.append('#9B149A')
              degree.append(F.degree(node))
    
  

    
      positions = force_atlas2_layout(F,
                                iterations=60,
                                pos_list=None,
                                node_masses=None,
                                outbound_attraction_distribution=False,
                                lin_log_mode=False,
                                prevent_overlapping=False,
                                edge_weight_influence=2.0,

                                jitter_tolerance=1.0,
                                barnes_hut_optimize=True,
                                barnes_hut_theta=1.2,

                                scaling_ratio=0.1,
                                strong_gravity_mode=False,
                                multithread=False,
                                gravity=10.0)

      options={
          'node_color':color_map,
          'node_size':degree,
          'width':0.4,
          'alpha':0.9
      }
    
  

    
      plt.figure(figsize=(30, 15))
      plt.title('Nutritional network')
      fig = plt.gcf()
      nx.draw(F, positions, **options)

      fig.set_size_inches(20, 15)
      plt.savefig('network')
    
  

    
      hist_dist={}
      nodesA = [x for x,y in F.nodes(data=True) if y['Role']=='Aliment']
      for ali in nodesA:
          hist_dist[ali]=F.degree[ali]
          
      count1,bins1=np.histogram(list(hist_dist.values()),bins=np.arange(0,51))
      plt.bar(bins1[:-1], count1,color='darkorange')
      plt.title('Distribution of degrees of the Aliments Nodes of our network',fontsize=10)
      plt.xlabel('Degree',fontsize=12,fontweight='bold')
      plt.ylabel('Count',fontsize=12)
      plt.savefig('Histogram')
      plt.show()
    
  

    
      sorted_hist = sorted(hist_dist.items(), key=lambda x: x[1], reverse=True)
      for i in range(1,11):
          print(i,'-',sorted_hist[i][0],'is rich in',sorted_hist[i][1],'features.')
    
  

    
      aliments_0=0
      for k,v in hist_dist.items():
          if v==0:
              aliments_0+=1
      print('There are',aliments_0,'with 0 edges.')
    
  

    
      def Girvan_Newman(G,num_communities=6):
      communities=1
      while communities < num_communities:
          d_s=nx.shortest_path(G)
          s=[]
          for i in d_s.values():
              for j in i.values():
                  if len(j)>1:
                      s.append(j)
          edges=[]
          for el in s:
              if len(el)>2:
                  l=len(el);c=0
                  while c < l-1:
                      edges.append(str(el[c:c+2]));c+=1
              else:
                  edges.append(str(el))
          counts=Counter(edges)
          d={}
          for el in list(G.edges()):
              idx=str(list(el))
              d[el]=counts[idx]

          # Remove edge with highest centrality
          idx_max=list(d.values()).index(max(d.values()))
          max_edge=list(d.keys())[idx_max]
          G.remove_edges_from([max_edge])
          communities+=1
      return G
    
  

    
      #Using a subgraph to apply the handed-coded Girwan_Newman
      sub_graph=sF.copy()
      remove = [node for node,degree in dict(sub_graph.degree()).items() if degree < 45]
      sub_graph.remove_nodes_from(remove)
      remove2 = [node for node,degree in dict(sub_graph.degree()).items() if degree ==0]
      sub_graph.remove_nodes_from(remove2)
      print('Initial subgraph with',len(sub_graph.nodes()),'nodes and',len(sub_graph.edges()),'edges.')

      nx.draw(sub_graph, with_labels=True,font_size=8)
    
  

    
      P=Girvan_Newman(sub_graph)
      print('New graph using Girvan_Newman Alorithm',P,'.')
      nx.draw(P, with_labels=True,font_size=8)
    
  

    
      partition = community.community_louvain.best_partition(sF)
      print('Number of communities found: ', len(np.unique(list(partition.values()))))
      mod = community.community_louvain.modularity(partition, sF)
      print("The modularity value is:","{:.3f}".format(mod))
    
  

    
      def get_keys_from_value(d, val):
      return [k for k, v in d.items() if v == val]

      #Delete the communities that have just one node:
      v=list(partition.values())
      p=partition
      a_remove=[]
      for i in (list(range(0,len(np.unique(list(partition.values())))))):
          x=list(partition.values()).count(i) 
          if x==1:
              v.remove(i)
              a=get_keys_from_value(p, i)
              a_remove.append(a[0])
              del p[a[0]]
              
              
      communities_d=dict.fromkeys(list(range(0,len(np.unique(v)))))

      sF2=sF.copy()
      sF2.remove_nodes_from(a_remove)

      # We compute the number of songs in every community
      for i in range(0,len(np.unique(v))):
          l=[]
          for el in partition:
              if partition[el]==list(np.unique(v))[i]:
                  l.append(el)
          communities_d[i]=l
          
      size_communities=[len(x) for x in communities_d.values()]
      for ind,j in enumerate(size_communities):
          print('Community',ind,'with:',j,'nodes.')
    
  

    
      #Delete the communities that have just one node to plot histogram:
      sub_communities_v=[]

      for i in range(0,len(size_communities)):
          sub_communities_v=sub_communities_v+[i]*size_communities[i]

      count,bins= np.histogram(sub_communities_v,bins=[0,1,2,3,4])
      plt.bar(bins[:-1], count,color='#0877aa');plt.title('Community Size Distribution',fontsize=13)
      plt.xlabel('Community',fontsize=12); plt.ylabel('Counts',fontsize=12)
      plt.savefig('size_distribution')
      plt.show()
    
  

    
      communities_named={}
      for el in communities_d:
          degrees=[sF2.degree(node) for node in communities_d[el]]
          communities_named[communities_d[el][degrees.index(max(degrees))]]=communities_d[el]

      for ind, j in enumerate(communities_named.keys()):
          print('Community',ind,'with',"\033[1m" + j + "\033[0m",'as the node with the maximum degree.')
    
  

    
      node_community = [node[1] for node in p.items()]
      pCol = {i: list(np.random.random(size=3)) for i in set(node_community)}

      positions = force_atlas2_layout(sF2,
                                iterations=100,
                                pos_list=None,
                                node_masses=None,
                                outbound_attraction_distribution=False,
                                lin_log_mode=False,
                                prevent_overlapping=False,
                                edge_weight_influence=2.0,

                                jitter_tolerance=1.0,
                                barnes_hut_optimize=True,
                                barnes_hut_theta=1.2,

                                scaling_ratio=0.1,
                                strong_gravity_mode=False,
                                multithread=False,
                                gravity=10.0)

      
      plt.figure(figsize=(30, 15))
      f = dict(nx.degree(sF2))
      node_sizes = [v*10 for v in f.values()]
      plt.title('Communities found in the Nutritional network')
      cmap = cm.get_cmap('viridis', len(np.unique(v)))
      fig = plt.gcf()
      nx.draw(sF2, positions, node_size=node_sizes, with_labels=False, width = 0.35, node_color=list(p.values()), alpha=0.9)
      # add legend
      legend=[]
      for idx,el in enumerate(cmap.colors):
          legend.append(mpatches.Patch(color=cmap.colors[idx], label=list(communities_named.keys())[idx]+' (size:'+str(size_communities[idx])+')'))
      plt.legend(handles=legend)
      fig.set_size_inches(20, 15)
      plt.savefig('communities_graph')
    
  

    
      #World Cloud to better define communities:
      wordcloud_dict={}
      for i in np.unique(v):
          a=get_keys_from_value(p, i)
          degree_a={}
          for node in a:
              degree_a[node]=sF2.degree(node)
              
          s_dict=dict(sorted(degree_a.items(), key=lambda item: item[1],reverse=True))
          text=[]
          for k,v in s_dict.items():
              text=text+[k]*v
          text_comm = ''
          for el in text:
              text_comm +=el+' '
          wordcloud_dict[i]= text_comm
          #Title of the network
      

      names=list(communities_named.keys())
      width = 1200; height = 2000
      wordcloud_0 = WordCloud(colormap = cm.OrRd, max_words = 100, width=width, height=height, background_color = "white",
                              collocations = False).generate(wordcloud_dict[list(wordcloud_dict.keys())[0]])
      wordcloud_1 = WordCloud(colormap = cm.OrRd, max_words = 100, width=width, height=height, background_color = "white",
                              collocations = False).generate(wordcloud_dict[list(wordcloud_dict.keys())[1]])
      wordcloud_2 = WordCloud(colormap = cm.OrRd, max_words = 100, width=width, height=height, background_color = "white",
                              collocations = False).generate(wordcloud_dict[list(wordcloud_dict.keys())[2]])
      wordcloud_3 = WordCloud(colormap = cm.OrRd, max_words = 100, width=width, height=height, background_color = "white",
                              collocations = False).generate(wordcloud_dict[list(wordcloud_dict.keys())[3]])


      fig, axs = plt.subplots(1, 4,figsize=(60,20))
      axs[0].imshow(wordcloud_0); axs[0].set_title(names[0],size=40); axs[0].axis('off')
      axs[1].imshow(wordcloud_1); axs[1].set_title(names[1],size=40); axs[1].axis('off')
      axs[2].imshow(wordcloud_2); axs[2].set_title(names[2],size=40); axs[2].axis('off')
      axs[3].imshow(wordcloud_3); axs[3].set_title(names[3],size=40); axs[3].axis('off')
      plt.savefig('Communities')
      plt.show()
    
  

    
      adj_matrix = nx.to_numpy_matrix(sF2) 
      node_list = list(sF2.nodes())

      #non-deterministic btw
      clusters = SpectralClustering(affinity='precomputed', n_clusters=4).fit_predict(adj_matrix)

      for i in np.unique(clusters):
      print('Cluster',i,'have',list(clusters).count(i),'elements.')
    
  

    
      def provide_aliments_rich_in_one_feature(feature,number_of_elements=10):
          d={}
          for el in F.edges(feature,data=True):
              d[el[1]]=el[2]['weight']
          features=list(d.keys())
          return features[0:number_of_elements]

      provide_aliments_rich_in_one_feature('sodium')
    
  

    
      deviation_percentage = 10;
      if consider macros == True:
          similar_macros = [Find foods in similar items that do not deviate more than 10% in macros levels]
              return elements in similar_macros that have the maximum number of features
      else:
          return elements in similar_items that have the maximum number of features
    
  

    
      return "According to features in which is rich, it is the best food element you can have."
    
  

    
      df=df_notnorm.copy()

      F= nx.Graph()
      for el in list(df):
          F.add_node(el,Role='Feature')
      for el in df.index:
          F.add_node(el,Role='Aliment')
          
      print("There are " +str(F.number_of_nodes())+" nodes in our network.")


      for f in df.columns:
          sorted_df=df.sort_values(by=[f],ascending=False).iloc[:criteria,0]
          for idx,el in enumerate(sorted_df.index):
              F.add_weighted_edges_from([(f,el,sorted_df[idx])])
    
  

    
      def draw_subgraph(food,micros_food,list_neigh2):
          new_feat=list(set(list_neigh2) - set(micros_food))
          S=nx.Graph()
          S.add_node(food, Role='Food')
          for el in micros_food:
              S.add_node(el,Role='Existing')

          for el in new_feat:
              S.add_node(el,Role='New')

          print("There are " +str(S.number_of_nodes())+" nodes. Macronutrients values correspond to the content in 100g of the food.")

          for el in micros_food:
              S.add_edge(el,food)
          for el in new_feat:
              S.add_edge(el,food)


          color_map=[]
          degree=[]
          for node in S.nodes:
              if S.nodes[node]['Role']=='Existing':
                  color_map.append('#9B149A')
                  degree.append(1000)
              elif S.nodes[node]['Role']=='New':
                  color_map.append('#157B87')
                  degree.append(1000)
              else:
                  color_map.append('yellow')
                  degree.append(5000)


          options={
              'node_color':color_map,
              'node_size':degree,
              'width':0.4,
              #'font_weight':'bold',
              'with_labels':True
          }
          plt.figure(figsize=(8,4),dpi=800)
          nx.draw(S,**options)
          plt.plot()
    
  

    
      def provide_better_substitutes(df_norm,F,food,consider_macros=True,deviation_percentage=10):
          macros=['carbohydrate', 'total_fat', 'protein']
          micros_food=[n for n in nx.neighbors(F, food)]
          # Get features the food is rich in (with weights)
          d={}
          for el in F.edges(food,data=True):
              d[el[1]]=el[2]['weight']
          features=list(d.keys())
          # We need a dataframe normalized
          index_feat=list(df_norm.loc[food,:]).index(max(df_norm.loc[food,:]))
          feature=df_norm.columns[index_feat]
          try:
              neigh= nx.neighbors(F, feature)
          except:
              neigh= nx.neighbors(F, 'calories') # in the hypothethical case there is an error, we go for calories as default
          list_neigh= [n for n in neigh]
          # Get list of nodes connected to this main feature with their corresponding neighbours
          d1={}
          for el in list_neigh:
              d1[el]=[el[1] for el in F.edges(el)]
          # Get subset of aliments that have (at least) the same amount of features as the food inputed
          features=list(d.keys())
          similar_items=[]
          for el in d1:
              if set(features).issubset(set(d1[el])):
                  similar_items.append(el)
          if len(similar_items)>1:
              """Consider Macros == True: we filter with this subset of food elements that do not provide
              a substantial amount of variation in the levels of macros"""
              food_macros=[df.loc[food][m] for m in macros]
              aliments=similar_items
              d={}
              for al in aliments:
                  mac=[]
                  for idx,m in enumerate(macros):
                      mac.append(food_macros[idx]-float(df.loc[al][m]))
                  d[al]=mac
              if consider_macros==True:
                  # Keep the ones that stand in +/- 10 grams of each macros
                  bool_filter=[]
                  for al in aliments:
                      var=True
                      for el in d[al]:
                          if abs(el)>deviation_percentage:
                              var=False
                      bool_filter.append(var)
                  similar_macros = [i for (i, v) in zip(aliments,[el == True for el in bool_filter]) if v]

                  if len(similar_macros)>1:
                      # Sort a dictionary with the elements that have more features
                      d3={}
                      for el in similar_macros:
                          d3[el]=len(F.edges(el))
                      sorted_d = dict(sorted(d3.items(), key=operator.itemgetter(1),reverse=True))
                      # Get elements that have the maximum number of features
                      filtered_list = [i for (i, v) in zip(list(sorted_d.keys()),[el == max(sorted_d.values()) for el in list(sorted_d.values())]) if v]
                      print('Recommended aliments:')
                      for el in filtered_list:
                          print("\033[1m" +"- "+ el + "\033[0m")
                          print(f'with deviations of {np.round(-d[el][0],2):+}g in carbohydrates, {np.round(-d[el][1],2):+}g in total fats and {-np.round(d[el][2],2):+}g in proteins.')
                      
                          # Get features that have been considered that before were not
                          list_neigh2= [n for n in nx.neighbors(F, el)]
                          draw_subgraph(el,micros_food,list_neigh2)
                      
                      return similar_macros,filtered_list
                  else:
                      # print('To consider macros you should increase the deviation percentage')
                      pass # We want to proceed as if consider macros was false
                  
              print('--- Suggested aliments might have discrepances in macros levels ---')
              # Sort a dictionary with the elements that have more features
              d3={}
              for el in similar_items:
                  d3[el]=len(F.edges(el))
              sorted_d = dict(sorted(d3.items(), key=operator.itemgetter(1),reverse=True))
              # Get elements that have the maximum number of features
              filtered_list = [i for (i, v) in zip(list(sorted_d.keys())
                                              ,[el == max(sorted_d.values()) for el in list(sorted_d.values())]) if v]
              
              print('Recommended aliments:');
              for el in filtered_list:
                  print("\033[1m" +"- "+ el + "\033[0m",end='\n')
                  print(f'with deviations of {np.round(-d[el][0],2):+}g in carbohydrates, {np.round(-d[el][1],2):+}g in total fats and {np.round(-d[el][2],2):+}g in proteins.',end='\n')
                  list_neigh2= [n for n in nx.neighbors(F, el)]
                  draw_subgraph(el,micros_food,list_neigh2)
              return similar_items,filtered_list

          else:
              print('According to features in which is rich, it is the best food element you can have.')
              return _,_
              pass
    
  

    
      s,f=provide_better_substitutes(df,F,'Fat, mutton tallow',consider_macros=True,deviation_percentage=10)
    
  

    
      s,f=provide_better_substitutes(df,F,'Fat, mutton tallow',consider_macros=False,deviation_percentage=10)
    
  

    
      s,f=provide_better_substitutes(df,F,'Nuts, pecans',consider_macros=True,deviation_percentage=10)
    
  

    
      s,f=provide_better_substitutes(df,F,'Cornstarch',consider_macros=True,deviation_percentage=10)
    
  

    
      s,f=provide_better_substitutes(df,F,'Eggplant, raw',consider_macros=False,deviation_percentage=10)
    
  

    
      s,f=provide_better_substitutes(df_norm,F,'Cauliflower, raw',consider_macros=False,deviation_percentage=10)