As a dirty hack, you could draw the circle two times: once with a solid line and after covering this up, a dashed line on top:


```
\documentclass[border=2mm,12pt,tikz]{standalone}
\usepackage{tikz-3dplot-circleofsphere}
\usepackage{fouriernc}
\usetikzlibrary{3dtools}

\begin{document}
    \tdplotsetmaincoords{60}{130}
    \begin{tikzpicture}[tdplot_main_coords,scale=1,line join = round, line cap = round,declare function={a=5;R=a*sqrt(2)/2;r=a*sqrt(3)/3; Angle=acos(r/R);}]
    \path
    (0,0,0) coordinate (A)
    (a,0,0) coordinate (B)
    (a/2,{a*sqrt(3)/2},0) coordinate (C)
   ($(A)!0.5! (C)$) coordinate (O)
    ($2*(O) - (B)$) coordinate (D)
  (a/2,{a*sqrt(3)/6},0) coordinate (H)
  (a/2,{a*sqrt(3)/6},{a*sqrt(6)/3}) coordinate (S)
  (0,{a*sqrt(3)/3},{a*sqrt(6)/6}) coordinate (I)
   ;
   
    \begin{scope}
         \draw[tdplot_screen_coords] (I) circle (R);
    \end{scope} 
       \begin{scope}[shift={(I)}]
       \tdplotCsDrawLatCircle[]{R}{{-Angle}}
       \end{scope} 
   
   \fill[white] (S) -- (C) -- (B) -- cycle;
   
   \foreach \p in {A,B,C,D,O,H,S,I}
   {\draw[fill=black] (\p) circle (1.5pt);}
   \foreach \p/\g in {A/60,B/-90,C/-90,D/0,O/-80,S/90,H/-90,I/90}
   {\path (\p)+(\g:3mm) node{$\p$};}
   \draw[dashed]  (A) -- (C) (B) -- (D) (A) -- (B) (A) -- (C) -- (D) -- cycle ;
   \foreach \X in {A,C,D,H} \draw [dashed] (S) -- (\X);
   \draw (B) -- (C) (S) -- (B);
   \begin{scope}
   \clip (S) -- (C) -- (B) -- cycle;
   \begin{scope}[shift={(I)}]
   \tdplotCsDrawLatCircle[tdplotCsFront/.style={dashed}]{R}{{-Angle}}
   \end{scope} 
    \begin{scope}
      \draw[tdplot_screen_coords,dashed] (I) circle (R);
 \end{scope} 
 \end{scope}
   \end{tikzpicture}
\end{document}
```

![document.png](/image?hash=2f25f1256e24d49facd5d050e199c5f79fa63b8e3dbb483b179b992d7656c874)

This saves and reuses paths either inside or outside the triangle. In order to accomplish the latter, an inverted clip is utilized, which borrowed from [stackexchange](https://tex.stackexchange.com/a/59168). 
```
\documentclass[border=2mm,12pt,tikz]{standalone}
\usepackage{tikz-3dplot-circleofsphere}
\usepackage{fouriernc}
% https://tex.stackexchange.com/a/59168
\tikzset{reverseclip/.style={overlay,insert path={(-12383.99999pt,-12383.99999pt) rectangle (12383.99999pt,12383.99999pt)}}}
\begin{document}
    \tdplotsetmaincoords{60}{130}
    \begin{tikzpicture}[tdplot_main_coords,scale=1,line join = round, line cap = round,declare function={a=5;R=a*sqrt(2)/2;r=a*sqrt(3)/3; Angle=acos(r/R);}]
    \path
    (0,0,0) coordinate (A)
    (a,0,0) coordinate (B)
    (a/2,{a*sqrt(3)/2},0) coordinate (C)
   ($(A)!0.5! (C)$) coordinate (O)
    ($2*(O) - (B)$) coordinate (D)
  (a/2,{a*sqrt(3)/6},0) coordinate (H)
  (a/2,{a*sqrt(3)/6},{a*sqrt(6)/3}) coordinate (S)
  (0,{a*sqrt(3)/3},{a*sqrt(6)/6}) coordinate (I)
   ;
   \foreach \p in {A,B,C,D,O,H,S,I}
   {\draw[fill=black] (\p) circle (1.5pt);}
   \foreach \p/\g in {A/60,B/-90,C/-90,D/0,O/-80,S/90,H/-90,I/90}
   {\path (\p)+(\g:3mm) node{$\p$}; }
   \draw[dashed]  (A) -- (C) (B) -- (D) (A) -- (B) (A) -- (C) -- (D) -- cycle ;
   \foreach \X in {A,C,D,H} \draw [dashed] (S) -- (\X);
   \draw (B) -- (C) (S) -- (B);
   \begin{scope}[shift={(I)}]
    \tdplotCsDrawLatCircle[tdplotCsFront/.style={draw=none,save path=\pathA}]{R}{{-Angle}}
   \end{scope} 
   \path[tdplot_screen_coords,save path=\pathB] (I) circle[radius=R];
   \begin{scope}
     \clip (B) -- (C) -- (S) [reverseclip];
   	 \draw[use path=\pathA];
	 \draw[use path=\pathB];
   \end{scope}
   \begin{scope}[dashed]
     \clip (B) -- (C) -- (S);
   	 \draw[use path=\pathA];
	 \draw[use path=\pathB];
   \end{scope}
   \end{tikzpicture}
\end{document}
```
![Screen Shot 2020-08-03 at 1.01.18 PM.png](/image?hash=c0fa307b5620a907fec76a607face1ad91db035fd2a3118aa299f4666132a819)