\begin{tikzpicture} [
    level 1/.style={sibling distance = 10em, level distance = 4em},
    level 2/.style={sibling distance = 5em},
    level 3/.style={sibling distance = 2.5em},
    tn/.style={inner sep=0, outer sep=0, line width=0.1em, minimum
      size =2em},
    root/.style={circle, draw=black, tn},
    on/.style={inner sep=0, outer sep=0, line width=0.02em, minimum size =2em},
    intree/.style={t/.style={shape=rectangle,tn}, n/.style={shape=circle, draw=black,tn}, 
      edge from parent/.style={black,line width=0.1cm,draw}}, 
    outtree/.style={t/.style={shape=rectangle,on}, n/.style={circle, draw=black!50,thin,
        on, text=black!50}, 
      edge from parent/.style={black!50,line width=0.01cm,draw}},
    scale=.8,
    every node/.style={scale=.8},
    payoff/.style={yshift=.5em, xshift=1em},
    vc/.style={xshift=1em, shape=circle, draw=black, line
      width=0.01em, xshift=-1em, yshift=-.5em, inner sep=0.1em},
  ]
  \begin{scope}
    \node[root] {} 
    child [intree] { node [n] {} edge from parent 
      child [intree] { node [n] {} edge from parent 
        child [intree] { node [n,t] {$0.5$} edge from parent node
          [vc,xshift=-.8em, yshift=.5em]
        {$3$} }
        child [intree] { node [n,t] {$0.5$} edge from parent node [vc,xshift=.8em, yshift=.5em]
        {$2$}}
        node [payoff] {$0.5$} node [vc,xshift=.3em] {$6$}
      }
      child [intree] { node [n] {} edge from parent 
        child [intree] { node [n,t] {$0.6$} edge from parent node [vc,
          xshift=-.8em, yshift=.5em]
        {$2$}}
        child [intree] { node [n,t] {$0.4$} edge from parent node [vc,
          xshift=.8em, yshift=.5em]
        {$2$}}
        node [payoff] {$0.5$} node [vc,xshift=-.3em] {$5$}
      }
      node [payoff,yshift=.5em] {$0.5$} node [vc,yshift=-.5em] {$12$}
    }
    child [intree] { node [n] {} edge from parent 
      child [intree] { node [n] {} edge from parent 
        child [outtree] { node [n,t] {$0.4$} edge from parent }
        child [outtree] { node [n,t] {$0.4$} edge from parent }
        node [payoff,xshift=-1.5em,yshift=.5em] {$0.4$} node [vc,xshift=-.8em,yshift=.3em] {$1$}
      }
      child [intree] { node [n] {} edge from parent 
        child [outtree] { node [n,t] {$0$} edge from parent }
        child [outtree] { node [n,t] {$0$} edge from parent }
        node [payoff] {$0$} node [vc,xshift=-.3em] {$1$}
      }
      node [payoff] {$0.13$} node [vc, xshift=-1em] {$3$}
    }
    child [intree, xshift=-2em] { node [n] {} edge from parent 
      child [outtree] { node [n,t] {$0.7$} edge from parent }
      child [outtree,xshift=-1em] { node [n] {} edge from parent 
        child [outtree] { node [n,t] {$0.1$} edge from parent }
        child [outtree] { node [n,t] {$0.3$} edge from parent }
        child [outtree] { node [n,t] {$0.1$} edge from parent }
        node [payoff] {$0.1$} }
      node (qex) [payoff] {$0.1$} node (vcex) [vc,xshift=-.5em] {$1$} 
    }
    ;
  \end{scope}

  \begin{scope}[xshift=15em]
    \node (qdesc) {Estimated reward};
    \node [below of=qdesc] (vcdesc) {Visit counter};
    \draw (qdesc) -- (qex);
    \draw (vcdesc) -- (vcex);
  \end{scope}
\end{tikzpicture}