aboutsummaryrefslogtreecommitdiffstats log msg author committer range
blob: a6ee38a770d26ef24147f26c85d6ed0be8a9e3bc (plain)
 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299  pre { line-height: 125%; margin: 0; } td.linenos pre { color: #000000; background-color: #f0f0f0; padding: 0 5px 0 5px; } span.linenos { color: #000000; background-color: #f0f0f0; padding: 0 5px 0 5px; } td.linenos pre.special { color: #000000; background-color: #ffffc0; padding: 0 5px 0 5px; } span.linenos.special { color: #000000; background-color: #ffffc0; padding: 0 5px 0 5px; } .highlight .hll { background-color: #ffffcc } .highlight .c { color: #888888 } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ .highlight .k { color: #008800; font-weight: bold } /* Keyword */ .highlight .ch { color: #888888 } /* Comment.Hashbang */ .highlight .cm { color: #888888 } /* Comment.Multiline */ .highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */ .highlight .cpf { color: #888888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */% LICENSE: see LICENCE \section{System Environment} In this section, we describe the system configuration requirements for the continuous operation of the tools. The production environment differs in some respects to those used for experimentation and in laboratory testing. Monitoring, automatic recovery (in case of errors) and resilience are crucial in 24/7 operations. The term \emph{production environment} will be used here to refer to such use. \subsection{Launching the tools} Services running in a production environment are usually administered remotely, and must be able to run without user intervention, or connection. Traditionally, such services are implemented (in UNIX terminology) as daemons'. These are started and stopped using the init system contained within the distribution. The ODR-mmbTools cannot daemonise themselves, so this requires another approach: \paragraph{Screen multiplexer} A simple approach is to use a screen multiplexer such as \emph{GNU Screen} or \emph{tmux} - either of these can be used to launch a session from which the user can detach, and later re-attach at will - leaving the tools running within it. Please see the relevant manpages for more information. Although a screen multiplexer alone permits the tools to run without a user being connected, it alone cannot automatically restart failed processes, and it is unable to provide warnings in the case of a problem. The dab-scripts, already mentioned in \ref{usingexistingwebstreams}, can be employed to monitor the processes and (if necessary) restart them, and send an alert via email. \paragraph{supervisord} As an alternative to using the scripts, the execution of the tools can also be carried out by a dedicated tool. \texttt{supervisor}\footnote{\url{http://supervisord.org}} is (as the name implies) such a tool. Once installed, supervisor reads its configuration file in \texttt{/etc/supervisor.conf} and launches the processes that are to be monitored. Each process is described by a file. The following example assumes the tools are run as user \texttt{odr}, and that the multiplex configuration is in \texttt{/home/odr/config.mux}, and that ODR-DabMux is to be launched.The logs of ODR-DabMux is written to the specified log files. \begin{lstlisting} [program:ODR-DabMux] command=odr-dabmux config.mux directory=/home/odr user=odr autostart=true autorestart=true stdout_logfile=/home/odr/logs/mux.out.log stderr_logfile=/home/odr/logs/mux.err.log \end{lstlisting} Once this configuration has been added to the supervisor configuration, the settings have to be re-read using: \begin{lstlisting} supervisorctl reread \end{lstlisting} In order for supervisor to start managing and running this process, it needs to be added: \begin{lstlisting} supervisorctl add ODR-DabMux \end{lstlisting} Setting up more processes (such as any of the other tools) can be easily achieved by customising the configuration template above. Examples are provided in the \texttt{mmbtools-aux} repository, under the \texttt{supervisor} folder - these need to be changed to reflect the paths that are in use on your system. supervisor also includes a small web-server that can display the state of the managed processes. It is enabled with the \verb+[inet_http_server]+ setting in the configuration file. \subsection{Logging} Collecting information about events is essential within a production environment. This information is essential forensic analysis, and tracing sources of trouble. This is achieved through the logging of important messages that can be sent by the tools. ODR-DabMux and ODR-DabMod both support logging to standard error, to a file and to the system logger \texttt{syslog}. Logging to syslog is the most flexible approach; log information can be forwarded over the network to a centralised logging server - where logs can then be filtered according to the priority of each message. Both tools log to the LOCAL0 facility which in turn can be redirected into an ODR-mmbtools specific log file. \sidenote{Describe rsyslog configuration} In order to avoid the log files from becomming undesirably large, \texttt{logrotate} should be set to rotate the files automatically. \sidenote{Describe logrotate configuration} \subsection{Timing} The ODR-mmbTools require the system time to be accurate in order for them to function correctly - this is especially important when running a SFN, but is also important for standalone transmitters when in a production environment. It is also important to remember that most receivers have a clock that is synchronised to the clock time which is being transmitted by the multiplex to which it has been tuned. The system needs to run a NTP client that synchronises the system time over the network. Correct synchronisation can be checked using the \texttt{lpeers} command of the \texttt{ntpq} tool. The magnitude of the offset should be below $10$\ms. The performance of the NTP synchronisation should also be monitored permanently during operation. \subsection{Monitoring using munin} The Munin\footnote{\url{http://munin-monitoring.org/}} monitoring tool can create graphs for essential system health parameters. It can also send emails if values transgress the defined bounds - this assists the operator in the assessment of system status, as well as the health of the services. In addition to basic system measurements like CPU, RAM and disk usage, NTP synchronisation, disk and network performance (and much more besides), there are also custom data sources for ODR-DabMux. These data sources include ZMQ input buffer monitoring (buffer level, underruns and overruns) and the peak audio input levels (mono, or stereo). This can be installed by copying \verb+doc/stats_dabmux_multi.py+ to \texttt{/etc/munin/plugins.d}. They require that the ODR-DabMux management server is enabled in the configuration, and will automatically generate the graphs for the subchannels used in the configuration. \subsection{Monitoring using Xymon} The xymon monitoring tool\footnote{\url{http://xymon.sourceforge.net/}} is used to monitor the health of many types of systems. It can present the results in text, tables and/or graphs. It supports the basic health checks directly out of the box, and can be extended with scripts to perform non-standard health checks. The default mode of operation is that clients retrieve data and send it to the xymon server, which interprets the results, displays them and generates alerts if thresholds are exceeded. An alert can be send in an e-mail, an SMS or a tweet. The Perl script \verb+retodrs.pl+\footnote{The script name stands for ''Retrieve Opendigitalradio Status}, retrieves the status and statistics of an Opendigitalradio service and it reports the results to xymon. The information is retrieved from the management server within ODR-DabMux. The information presented includes a table with the status of each sub-channel and the underrun and overrun rates on the sub-channels. If needed an alert can be generated depending on the subchannel status or a rate exceeding a threshold. The script needs to be installed on the same server running ODR-DABmux, as the management service within it is only accessible from the same computer. This implies that the xymon client software also needs to be installed on the same machine. The client is configured to run the script. The configuration and the scripts can typically be found in subdirectory \verb+/usr/lib/xymon/client+, although that may depend on your distribution. Once the client is set up, it needs to connect to a xymon server, which may or may not be on the same machine. The server is configured to limit the altering to specific sub-channels, to store the statistical data and to generate graphs. The configuration and the scripts on a xymon server are usually stored in the subdirectory \verb+/usr/lib/xymon/server+. \subsubsection{Installation of the Xymon Client} The perl script has additional requirements: \texttt{App::cpanminus}, \texttt{ZMQ::LibZMQ3}, and \texttt{JSON::PP}. They can be installed through your distribution packages or using CPAN. Once the script has been copied to \verb+/usr/lib/xymon/client/ext+, the configuration of the launcher within the xymon client needs to be extended. Create a new file named \verb+odrmux.cfg+ in \verb+/usr/lib/xymon/client/etc/clientlaunch.d+ containing the following lines: \begin{verbatim} # # Test odrmux checks the state and the statistics # of the ODR-DabMux service. # [odrmux] ENVFILE $XYMONCLIENTHOME/etc/xymonclient.cfg CMD$XYMONCLIENTHOME/ext/retodrs.pl LOGFILE $XYMONCLIENTLOGS/retodrs.plog INTERVAL 5m \end{verbatim} After a restart of the xymon client, the script \verb+retodrs.pl+ will be invoked once every 5 minutes. \subsubsection{Server Configuration} By default all subchannels will be monitored, and will raise alerts if the status or the statistics are in outside of a valid operational range. The alerting can be limited to a subset of the sub-channels by adding a tag to the hosts-entry in the configuration file \verb+/usr/lib/xymon/server/etc/hosts.cfg+. The additional tag is: \begin{verbatim} ODR:select(;;...) \end{verbatim} The sub-channels not named will still be shown, but no alerts will be generated for those sub-channels. This is visible as the green/yellow/red icons are missing for those sub-channels. Six statistic values are gathered by the script, namely \texttt{BufferMin}, \texttt{BufferMax}, \texttt{PeakLeft}, \texttt{PeakRight}, \texttt{UnderRun} and \texttt{OverRun}. It is found that only the latter two seem to contain sensible values all the time, so those values are the only ones shown in a graph. Note that those values retrieved by the script are ever-increasing counters, showing the total number of over-runs or under-runs. In the graph, the average number of over-runs or under-runs per second, averaged over a period of 5 minutes, is shown. The first step is to have the collected statistics to be moved into a database, a so-called \textit{Round Robin Database}. This is accomplished by adding a file named \verb+odr.cfg+ in \verb+/usr/lib/xymon/server/etc/xymonserver.d+ containing the following lines: \begin{verbatim} TEST2RRD+=",odr_mux=devmon" GRAPHS+=",odr_mux::1" \end{verbatim} The next step is to define the layout of the graph. Create a file named \verb+graphs.odr.cfg+ in \verb+/usr/lib/xymon/server/etc/graphs.d+ containing the following lines: \begin{verbatim} # # Graphs to show the statistics collected from an # Opendigitalradio DabMux server. # [odr_mux] FNPATTERN ^odr_mux\.(.+)\.rrd$ TITLE , Frame loss rate YAXIS Rate [/s] -l 0 DEF:ur@RRDIDX@=@RRDFN@:Underrun:AVERAGE DEF:or@RRDIDX@=@RRDFN@:Overrun:AVERAGE LINE1:ur@RRDIDX@#FF0000:@RRDPARAM@ underrun GPRINT:ur@RRDIDX@:MIN:Min \: %5.1lf %s GPRINT:ur@RRDIDX@:MAX:Max \: %5.1lf %s GPRINT:ur@RRDIDX@:AVERAGE:Avg \: %5.1lf %s GPRINT:ur@RRDIDX@:LAST:Cur \: %5.1lf %s\n LINE1:or@RRDIDX@#00FF00:@RRDPARAM@ overrun GPRINT:or@RRDIDX@:MIN: Min \: %5.1lf %s GPRINT:or@RRDIDX@:MAX:Max \: %5.1lf %s GPRINT:or@RRDIDX@:AVERAGE:Avg \: %5.1lf %s GPRINT:or@RRDIDX@:LAST:Cur \: %5.1lf %s\n \end{verbatim} \subsection{Real-time Scheduling} As a general principle, it is prudent not to run tools (that do not need superuser privileges) as the \texttt{root} user. The same principle also applies to the ODR-mmbTools, but care has to be taken that the tools can still request real-time scheduling when it is needed. This is achieved by adding the following to \texttt{/etc/security/limits.conf}, assuming the tools are run under the user \texttt{odr}. \begin{lstlisting} odr - rtprio 65 odr - nice -10 \end{lstlisting} If you have installed JACK with real-time privileges, you may find this has already been configured for the audio' group, written as \texttt{@audio}, which should suffice providing your desired user is a member of the audio' group. \subsection{Accessing the USRP as Non-root} Superuser privileges are not required to access USB-connected USRP devices, but sometimes the system lacks the configuration to enable normal users to communicate with the device. In that case, it is necessary to add a rule file for \texttt{udev}. This file is included in the UHD sources, but might not have been automatically installed. The file is called \texttt{10-uhd-usrp.rules}, should be placed into \texttt{/etc/udev/rules.d/} and should contain { \footnotesize \begin{verbatim} #USRP1 SUBSYSTEMS=="usb", ATTRS{idVendor}=="fffe", ATTRS{idProduct}=="0002", MODE:="0666" #B100 SUBSYSTEMS=="usb", ATTRS{idVendor}=="2500", ATTRS{idProduct}=="0002", MODE:="0666" #B200 SUBSYSTEMS=="usb", ATTRS{idVendor}=="2500", ATTRS{idProduct}=="0020", MODE:="0666" \end{verbatim} } % vim: spl=en spell tw=80 et `