xref: /OpenGrok/opengrok-indexer/src/test/resources/analysis/scala/sample_xref.html (revision a4d08de4dc8d883b6857e1cb31b6128c283fe674) 
1*a4d08de4SAdam Hornáček<!DOCTYPE html>
2*a4d08de4SAdam Hornáček<html lang="en">
3eeb7e5b3SAdam Hornáček<head>
4*a4d08de4SAdam Hornáček<meta charset="UTF-8">
5eeb7e5b3SAdam Hornáček<title>sampleFile - OpenGrok cross reference for /sampleFile</title></head><body>
6eeb7e5b3SAdam Hornáček<script type="text/javascript">/* <![CDATA[ */
7eeb7e5b3SAdam Hornáčekfunction get_sym_list(){return [["Package","xp",[["org.apache.spark.broadcast",18]]],["Method","xmt",[["blockifyObject",270],["unBlockifyObject",287],["unary_",318],["unpersist",308]]]];} /* ]]> */</script><a class="l" name="1" href="#1">1</a><span class="c">/*</span>
8eeb7e5b3SAdam Hornáček<a class="l" name="2" href="#2">2</a><span class="c"> * Licensed to the Apache Software Foundation (ASF) under one or more</span>
9eeb7e5b3SAdam Hornáček<a class="l" name="3" href="#3">3</a><span class="c"> * contributor license agreements.  See the NOTICE file distributed with</span>
10eeb7e5b3SAdam Hornáček<a class="l" name="4" href="#4">4</a><span class="c"> * this work for additional information regarding copyright ownership.</span>
11eeb7e5b3SAdam Hornáček<a class="l" name="5" href="#5">5</a><span class="c"> * The ASF licenses this file to You under the Apache License, Version 2.0</span>
12eeb7e5b3SAdam Hornáček<a class="l" name="6" href="#6">6</a><span class="c"> * (the &quot;License&quot;); you may not use this file except in compliance with</span>
13eeb7e5b3SAdam Hornáček<a class="l" name="7" href="#7">7</a><span class="c"> * the License.  You may obtain a copy of the License at</span>
14eeb7e5b3SAdam Hornáček<a class="l" name="8" href="#8">8</a><span class="c"> *</span>
15eeb7e5b3SAdam Hornáček<a class="l" name="9" href="#9">9</a><span class="c"> *    <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a></span>
16eeb7e5b3SAdam Hornáček<a class="hl" name="10" href="#10">10</a><span class="c"> *</span>
17eeb7e5b3SAdam Hornáček<a class="l" name="11" href="#11">11</a><span class="c"> * Unless required by applicable law or agreed to in writing, software</span>
18eeb7e5b3SAdam Hornáček<a class="l" name="12" href="#12">12</a><span class="c"> * distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
19eeb7e5b3SAdam Hornáček<a class="l" name="13" href="#13">13</a><span class="c"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
20eeb7e5b3SAdam Hornáček<a class="l" name="14" href="#14">14</a><span class="c"> * See the License for the specific language governing permissions and</span>
21eeb7e5b3SAdam Hornáček<a class="l" name="15" href="#15">15</a><span class="c"> * limitations under the License.</span>
22eeb7e5b3SAdam Hornáček<a class="l" name="16" href="#16">16</a><span class="c"> */</span>
23eeb7e5b3SAdam Hornáček<a class="l" name="17" href="#17">17</a>
24eeb7e5b3SAdam Hornáček<a class="l" name="18" href="#18">18</a><b>package</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=broadcast" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcast</a>
25eeb7e5b3SAdam Hornáček<a class="l" name="19" href="#19">19</a>
26eeb7e5b3SAdam Hornáček<a class="hl" name="20" href="#20">20</a><b>import</b> <a href="/source/s?defs=java" class="intelliWindow-symbol" data-definition-place="undefined-in-file">java</a>.<a href="/source/s?defs=io" class="intelliWindow-symbol" data-definition-place="undefined-in-file">io</a>.<b>_</b>
27eeb7e5b3SAdam Hornáček<a class="l" name="21" href="#21">21</a><b>import</b> <a href="/source/s?defs=java" class="intelliWindow-symbol" data-definition-place="undefined-in-file">java</a>.<a href="/source/s?defs=nio" class="intelliWindow-symbol" data-definition-place="undefined-in-file">nio</a>.<a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a>
28eeb7e5b3SAdam Hornáček<a class="l" name="22" href="#22">22</a><b>import</b> <a href="/source/s?defs=java" class="intelliWindow-symbol" data-definition-place="undefined-in-file">java</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=zip" class="intelliWindow-symbol" data-definition-place="undefined-in-file">zip</a>.<a href="/source/s?defs=Adler32" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Adler32</a>
29eeb7e5b3SAdam Hornáček<a class="l" name="23" href="#23">23</a>
30eeb7e5b3SAdam Hornáček<a class="l" name="24" href="#24">24</a><b>import</b> <a href="/source/s?defs=scala" class="intelliWindow-symbol" data-definition-place="undefined-in-file">scala</a>.<a href="/source/s?defs=collection" class="intelliWindow-symbol" data-definition-place="undefined-in-file">collection</a>.<a href="/source/s?defs=JavaConverters" class="intelliWindow-symbol" data-definition-place="undefined-in-file">JavaConverters</a>.<b>_</b>
31eeb7e5b3SAdam Hornáček<a class="l" name="25" href="#25">25</a><b>import</b> <a href="/source/s?defs=scala" class="intelliWindow-symbol" data-definition-place="undefined-in-file">scala</a>.<a href="/source/s?defs=reflect" class="intelliWindow-symbol" data-definition-place="undefined-in-file">reflect</a>.<a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a>
32eeb7e5b3SAdam Hornáček<a class="l" name="26" href="#26">26</a><b>import</b> <a href="/source/s?defs=scala" class="intelliWindow-symbol" data-definition-place="undefined-in-file">scala</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=Random" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Random</a>
33eeb7e5b3SAdam Hornáček<a class="l" name="27" href="#27">27</a>
34eeb7e5b3SAdam Hornáček<a class="l" name="28" href="#28">28</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<b>_</b>
35eeb7e5b3SAdam Hornáček<a class="l" name="29" href="#29">29</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=internal" class="intelliWindow-symbol" data-definition-place="undefined-in-file">internal</a>.<a href="/source/s?defs=Logging" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Logging</a>
36eeb7e5b3SAdam Hornáček<a class="hl" name="30" href="#30">30</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=io" class="intelliWindow-symbol" data-definition-place="undefined-in-file">io</a>.<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>
37eeb7e5b3SAdam Hornáček<a class="l" name="31" href="#31">31</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>.<a href="/source/s?defs=Serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializer</a>
38eeb7e5b3SAdam Hornáček<a class="l" name="32" href="#32">32</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=storage" class="intelliWindow-symbol" data-definition-place="undefined-in-file">storage</a>.<b>_</b>
39eeb7e5b3SAdam Hornáček<a class="l" name="33" href="#33">33</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>
40eeb7e5b3SAdam Hornáček<a class="l" name="34" href="#34">34</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=io" class="intelliWindow-symbol" data-definition-place="undefined-in-file">io</a>.{<a href="/source/s?defs=ChunkedByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBuffer</a>, <a href="/source/s?defs=ChunkedByteBufferOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBufferOutputStream</a>}
41eeb7e5b3SAdam Hornáček<a class="l" name="35" href="#35">35</a>
42eeb7e5b3SAdam Hornáček<a class="l" name="36" href="#36">36</a><span class="c">/**</span>
43eeb7e5b3SAdam Hornáček<a class="l" name="37" href="#37">37</a><span class="c"> * A BitTorrent-like implementation of [[org.apache.spark.broadcast.Broadcast]].</span>
44eeb7e5b3SAdam Hornáček<a class="l" name="38" href="#38">38</a><span class="c"> *</span>
45eeb7e5b3SAdam Hornáček<a class="l" name="39" href="#39">39</a><span class="c"> * The mechanism is as follows:</span>
46eeb7e5b3SAdam Hornáček<a class="hl" name="40" href="#40">40</a><span class="c"> *</span>
47eeb7e5b3SAdam Hornáček<a class="l" name="41" href="#41">41</a><span class="c"> * The driver divides the serialized object into small chunks and</span>
48eeb7e5b3SAdam Hornáček<a class="l" name="42" href="#42">42</a><span class="c"> * stores those chunks in the BlockManager of the driver.</span>
49eeb7e5b3SAdam Hornáček<a class="l" name="43" href="#43">43</a><span class="c"> *</span>
50eeb7e5b3SAdam Hornáček<a class="l" name="44" href="#44">44</a><span class="c"> * On each executor, the executor first attempts to fetch the object from its BlockManager. If</span>
51eeb7e5b3SAdam Hornáček<a class="l" name="45" href="#45">45</a><span class="c"> * it does not exist, it then uses remote fetches to fetch the small chunks from the driver <a href="/source/s?path=and/">and</a>/<a href="/source/s?path=and/or">or</a></span>
52eeb7e5b3SAdam Hornáček<a class="l" name="46" href="#46">46</a><span class="c"> * other executors if available. Once it gets the chunks, it puts the chunks in its own</span>
53eeb7e5b3SAdam Hornáček<a class="l" name="47" href="#47">47</a><span class="c"> * BlockManager, ready for other executors to fetch from.</span>
54eeb7e5b3SAdam Hornáček<a class="l" name="48" href="#48">48</a><span class="c"> *</span>
55eeb7e5b3SAdam Hornáček<a class="l" name="49" href="#49">49</a><span class="c"> * This prevents the driver from being the bottleneck in sending out multiple copies of the</span>
56eeb7e5b3SAdam Hornáček<a class="hl" name="50" href="#50">50</a><span class="c"> * broadcast data (one per executor).</span>
57eeb7e5b3SAdam Hornáček<a class="l" name="51" href="#51">51</a><span class="c"> *</span>
58eeb7e5b3SAdam Hornáček<a class="l" name="52" href="#52">52</a><span class="c"> * When initialized, TorrentBroadcast objects read <a href="/source/s?path=SparkEnv.get.conf">SparkEnv.get.conf</a>.</span>
59eeb7e5b3SAdam Hornáček<a class="l" name="53" href="#53">53</a><span class="c"> *</span>
60eeb7e5b3SAdam Hornáček<a class="l" name="54" href="#54">54</a><span class="c"> * <strong>@param</strong> <em>obj</em> object to broadcast</span>
61eeb7e5b3SAdam Hornáček<a class="l" name="55" href="#55">55</a><span class="c"> * <strong>@param</strong> <em>id</em> A unique identifier for the broadcast variable.</span>
62eeb7e5b3SAdam Hornáček<a class="l" name="56" href="#56">56</a><span class="c"> */</span>
63eeb7e5b3SAdam Hornáček<a class="l" name="57" href="#57">57</a><b>private</b>[<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>] <b>class</b> <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>: <a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a>](<a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>, <a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>: <a href="/source/s?defs=Long" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Long</a>)
64eeb7e5b3SAdam Hornáček<a class="l" name="58" href="#58">58</a>  <b>extends</b> <a href="/source/s?defs=Broadcast" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Broadcast</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>](<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>) <b>with</b> <a href="/source/s?defs=Logging" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Logging</a> <b>with</b> <a href="/source/s?defs=Serializable" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializable</a> {
65eeb7e5b3SAdam Hornáček<a class="l" name="59" href="#59">59</a>
66eeb7e5b3SAdam Hornáček<a class="hl" name="60" href="#60">60</a>  <span class="c">/**</span>
67eeb7e5b3SAdam Hornáček<a class="l" name="61" href="#61">61</a><span class="c">   * Value of the broadcast object on executors. This is reconstructed by [[readBroadcastBlock]],</span>
68eeb7e5b3SAdam Hornáček<a class="l" name="62" href="#62">62</a><span class="c">   * which builds this value by reading blocks from the driver <a href="/source/s?path=and/">and</a>/<a href="/source/s?path=and/or">or</a> other executors.</span>
69eeb7e5b3SAdam Hornáček<a class="l" name="63" href="#63">63</a><span class="c">   *</span>
70eeb7e5b3SAdam Hornáček<a class="l" name="64" href="#64">64</a><span class="c">   * On the driver, if the value is required, it is read lazily from the block manager.</span>
71eeb7e5b3SAdam Hornáček<a class="l" name="65" href="#65">65</a><span class="c">   */</span>
72eeb7e5b3SAdam Hornáček<a class="l" name="66" href="#66">66</a>  @<a href="/source/s?defs=transient" class="intelliWindow-symbol" data-definition-place="undefined-in-file">transient</a> <b>private</b> <b>lazy</b> <b>val</b> <a href="/source/s?defs=_value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_value</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a> = <a href="/source/s?defs=readBroadcastBlock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBroadcastBlock</a>()
73eeb7e5b3SAdam Hornáček<a class="l" name="67" href="#67">67</a>
74eeb7e5b3SAdam Hornáček<a class="l" name="68" href="#68">68</a>  <span class="c">/** The compression codec to use, or None if compression is disabled */</span>
75eeb7e5b3SAdam Hornáček<a class="l" name="69" href="#69">69</a>  @<a href="/source/s?defs=transient" class="intelliWindow-symbol" data-definition-place="undefined-in-file">transient</a> <b>private</b> <b>var</b> <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>: <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>[<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>] = <b>_</b>
76eeb7e5b3SAdam Hornáček<a class="hl" name="70" href="#70">70</a>  <span class="c">/** Size of each block. Default value is 4MB.  This value is only read by the broadcaster. */</span>
77eeb7e5b3SAdam Hornáček<a class="l" name="71" href="#71">71</a>  @<a href="/source/s?defs=transient" class="intelliWindow-symbol" data-definition-place="undefined-in-file">transient</a> <b>private</b> <b>var</b> <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>: <a href="/source/s?defs=Long" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Long</a> = <b>_</b>
78eeb7e5b3SAdam Hornáček<a class="l" name="72" href="#72">72</a>
79eeb7e5b3SAdam Hornáček<a class="l" name="73" href="#73">73</a>  <b>private</b> <b>def</b> <a href="/source/s?defs=setConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">setConf</a>(<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>: <a href="/source/s?defs=SparkConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkConf</a>) {
80eeb7e5b3SAdam Hornáček<a class="l" name="74" href="#74">74</a>    <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a> = <b>if</b> (<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>.<a href="/source/s?defs=getBoolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getBoolean</a>(<span class="s">&quot;spark.broadcast.compress&quot;</span>, <b>true</b>)) {
81eeb7e5b3SAdam Hornáček<a class="l" name="75" href="#75">75</a>      <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>.<a href="/source/s?defs=createCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">createCodec</a>(<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>))
82eeb7e5b3SAdam Hornáček<a class="l" name="76" href="#76">76</a>    } <b>else</b> {
83eeb7e5b3SAdam Hornáček<a class="l" name="77" href="#77">77</a>      <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a>
84eeb7e5b3SAdam Hornáček<a class="l" name="78" href="#78">78</a>    }
85eeb7e5b3SAdam Hornáček<a class="l" name="79" href="#79">79</a>    <span class="c">// Note: use getSizeAsKb (not bytes) to maintain compatibility if no units are provided</span>
86eeb7e5b3SAdam Hornáček<a class="hl" name="80" href="#80">80</a>    <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a> = (<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>.<a href="/source/s?defs=getSizeAsKb" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getSizeAsKb</a>(<span class="s">&quot;spark.broadcast.blockSize&quot;</span>, <span class="s">&quot;4m&quot;</span>).<a href="/source/s?defs=toInt" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toInt</a> *
87eeb7e5b3SAdam Hornáček<a class="l" name="81" href="#81">81</a>        <span class="n">1024</span> + <span class="n">21</span> - <span class="n">21</span> + <span class="n">0xFE</span> - <span class="n">0xfe</span> + <span class="n">-42L</span> - <span class="n">42l</span> + <span class="n">0.0</span> + <span class="n">1e3f</span> - <span class="n">1e3f</span> +
88eeb7e5b3SAdam Hornáček<a class="l" name="82" href="#82">82</a>        <span class="n">3.0f</span> - <span class="n">3.0f</span> + <span class="n">-0.0e-2</span> + <span class="n">.1</span> - <span class="n">.1</span>)
89eeb7e5b3SAdam Hornáček<a class="l" name="83" href="#83">83</a>
90eeb7e5b3SAdam Hornáček<a class="l" name="84" href="#84">84</a>    <a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a> = <a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>.<a href="/source/s?defs=getBoolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getBoolean</a>(<span class="s">&quot;spark.broadcast.checksum&quot;</span>, <b>true</b>)
91eeb7e5b3SAdam Hornáček<a class="l" name="85" href="#85">85</a>  }
92eeb7e5b3SAdam Hornáček<a class="l" name="86" href="#86">86</a>  <a href="/source/s?defs=setConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">setConf</a>(<a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>)
93eeb7e5b3SAdam Hornáček<a class="l" name="87" href="#87">87</a>
94eeb7e5b3SAdam Hornáček<a class="l" name="88" href="#88">88</a>  <b>private</b> <b>val</b> <a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a> = <a href="/source/s?defs=BroadcastBlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BroadcastBlockId</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>)
95eeb7e5b3SAdam Hornáček<a class="l" name="89" href="#89">89</a>
96eeb7e5b3SAdam Hornáček<a class="hl" name="90" href="#90">90</a>  <span class="c">/** Total number of blocks this broadcast variable contains. */</span>
97eeb7e5b3SAdam Hornáček<a class="l" name="91" href="#91">91</a>  <b>private</b> <b>val</b> <a href="/source/s?defs=numBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">numBlocks</a>: <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a> = <a href="/source/s?defs=writeBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeBlocks</a>(<a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>)
98eeb7e5b3SAdam Hornáček<a class="l" name="92" href="#92">92</a>
99eeb7e5b3SAdam Hornáček<a class="l" name="93" href="#93">93</a>  <span class="c">/** Whether to generate checksum for blocks or not. */</span>
100eeb7e5b3SAdam Hornáček<a class="l" name="94" href="#94">94</a>  <b>private</b> <b>var</b> <a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a> = <b>false</b>
101eeb7e5b3SAdam Hornáček<a class="l" name="95" href="#95">95</a>  <span class="c">/** The checksum for all the blocks. */</span>
102eeb7e5b3SAdam Hornáček<a class="l" name="96" href="#96">96</a>  <b>private</b> <b>var</b> <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a>: <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a>] = <b>_</b>
103eeb7e5b3SAdam Hornáček<a class="l" name="97" href="#97">97</a>
104eeb7e5b3SAdam Hornáček<a class="l" name="98" href="#98">98</a>  <b>override</b> <b>protected</b> <b>def</b> <a href="/source/s?defs=getValue" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getValue</a>() = {
105eeb7e5b3SAdam Hornáček<a class="l" name="99" href="#99">99</a>    <a href="/source/s?defs=_value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_value</a>
106eeb7e5b3SAdam Hornáček<a class="hl" name="100" href="#100">100</a>  }
107eeb7e5b3SAdam Hornáček<a class="l" name="101" href="#101">101</a>
108eeb7e5b3SAdam Hornáček<a class="l" name="102" href="#102">102</a>  <b>private</b> <b>def</b> <a href="/source/s?defs=calcChecksum" class="intelliWindow-symbol" data-definition-place="undefined-in-file">calcChecksum</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>: <a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a>): <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a> = {
109eeb7e5b3SAdam Hornáček<a class="l" name="103" href="#103">103</a>    <b>val</b> <a class="d" name="adler"/><a href="/source/s?refs=adler" class="d intelliWindow-symbol" data-definition-place="def">adler</a> = <b>new</b> <a href="/source/s?defs=Adler32" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Adler32</a>()
110eeb7e5b3SAdam Hornáček<a class="l" name="104" href="#104">104</a>    <b>if</b> (<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=hasArray" class="intelliWindow-symbol" data-definition-place="undefined-in-file">hasArray</a>) {
111eeb7e5b3SAdam Hornáček<a class="l" name="105" href="#105">105</a>      <a class="d intelliWindow-symbol" href="#adler" data-definition-place="defined-in-file">adler</a>.<a href="/source/s?defs=update" class="intelliWindow-symbol" data-definition-place="undefined-in-file">update</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">array</a>, <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=arrayOffset" class="intelliWindow-symbol" data-definition-place="undefined-in-file">arrayOffset</a> + <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=position" class="intelliWindow-symbol" data-definition-place="undefined-in-file">position</a>(), <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=limit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">limit</a>()
112eeb7e5b3SAdam Hornáček<a class="l" name="106" href="#106">106</a>        - <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=position" class="intelliWindow-symbol" data-definition-place="undefined-in-file">position</a>())
113eeb7e5b3SAdam Hornáček<a class="l" name="107" href="#107">107</a>    } <b>else</b> {
114eeb7e5b3SAdam Hornáček<a class="l" name="108" href="#108">108</a>      <b>val</b> <a class="d" name="bytes"/><a href="/source/s?refs=bytes" class="d intelliWindow-symbol" data-definition-place="def">bytes</a> = <b>new</b> <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=Byte" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Byte</a>](<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=remaining" class="intelliWindow-symbol" data-definition-place="undefined-in-file">remaining</a>())
115eeb7e5b3SAdam Hornáček<a class="l" name="109" href="#109">109</a>      <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=duplicate" class="intelliWindow-symbol" data-definition-place="undefined-in-file">duplicate</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>(<a href="/source/s?defs=bytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">bytes</a>)
116eeb7e5b3SAdam Hornáček<a class="hl" name="110" href="#110">110</a>      <a class="d intelliWindow-symbol" href="#adler" data-definition-place="defined-in-file">adler</a>.<a href="/source/s?defs=update" class="intelliWindow-symbol" data-definition-place="undefined-in-file">update</a>(<a href="/source/s?defs=bytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">bytes</a>)
117eeb7e5b3SAdam Hornáček<a class="l" name="111" href="#111">111</a>    }
118eeb7e5b3SAdam Hornáček<a class="l" name="112" href="#112">112</a>    <a class="d intelliWindow-symbol" href="#adler" data-definition-place="defined-in-file">adler</a>.<a href="/source/s?defs=getValue" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getValue</a>.<a href="/source/s?defs=toInt" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toInt</a>
119eeb7e5b3SAdam Hornáček<a class="l" name="113" href="#113">113</a>  }
120eeb7e5b3SAdam Hornáček<a class="l" name="114" href="#114">114</a>
121eeb7e5b3SAdam Hornáček<a class="l" name="115" href="#115">115</a>  <span class="c">/**</span>
122eeb7e5b3SAdam Hornáček<a class="l" name="116" href="#116">116</a><span class="c">   * Divide the object into multiple blocks and put those blocks in the block manager.</span>
123eeb7e5b3SAdam Hornáček<a class="l" name="117" href="#117">117</a><span class="c">   *</span>
124eeb7e5b3SAdam Hornáček<a class="l" name="118" href="#118">118</a><span class="c">   * <strong>@param</strong> <em>value</em> the object to divide</span>
125eeb7e5b3SAdam Hornáček<a class="l" name="119" href="#119">119</a><span class="c">   * <strong>@return</strong> number of blocks this broadcast variable is divided into</span>
126eeb7e5b3SAdam Hornáček<a class="hl" name="120" href="#120">120</a><span class="c">   */</span>
127eeb7e5b3SAdam Hornáček<a class="l" name="121" href="#121">121</a>  <b>private</b> <b>def</b> <a href="/source/s?defs=writeBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeBlocks</a>(<a href="/source/s?defs=value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">value</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>): <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a> = {
128eeb7e5b3SAdam Hornáček<a class="l" name="122" href="#122">122</a>    <b>import</b> <a href="/source/s?defs=StorageLevel" class="intelliWindow-symbol" data-definition-place="undefined-in-file">StorageLevel</a>.<b>_</b>
129eeb7e5b3SAdam Hornáček<a class="l" name="123" href="#123">123</a>    <span class="c">// Store a copy of the broadcast variable in the driver so that tasks run on the driver</span>
130eeb7e5b3SAdam Hornáček<a class="l" name="124" href="#124">124</a>    <span class="c">// do not create a duplicate copy of the broadcast variable&apos;s value.</span>
131eeb7e5b3SAdam Hornáček<a class="l" name="125" href="#125">125</a>    <b>val</b> <a class="d" name="blockManager"/><a href="/source/s?refs=blockManager" class="d intelliWindow-symbol" data-definition-place="def">blockManager</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>
132eeb7e5b3SAdam Hornáček<a class="l" name="126" href="#126">126</a>    <b>if</b> (!<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=putSingle" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putSingle</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>, <a href="/source/s?defs=value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">value</a>, <a href="/source/s?defs=MEMORY_AND_DISK" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>false</b>)) {
133eeb7e5b3SAdam Hornáček<a class="l" name="127" href="#127">127</a>      <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">&quot;Failed to store $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in BlockManager&quot;</span>)
134eeb7e5b3SAdam Hornáček<a class="l" name="128" href="#128">128</a>    }
135eeb7e5b3SAdam Hornáček<a class="l" name="129" href="#129">129</a>    <b>val</b> <a class="d" name="blocks"/><a href="/source/s?refs=blocks" class="d intelliWindow-symbol" data-definition-place="def">blocks</a> =
136eeb7e5b3SAdam Hornáček<a class="hl" name="130" href="#130">130</a>      <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#blockifyObject" data-definition-place="defined-in-file">blockifyObject</a>(<a href="/source/s?defs=value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">value</a>, <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>, <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>, <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>)
137eeb7e5b3SAdam Hornáček<a class="l" name="131" href="#131">131</a>    <b>if</b> (<a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>) {
138eeb7e5b3SAdam Hornáček<a class="l" name="132" href="#132">132</a>      <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a> = <b>new</b> <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a>](<a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=length" class="intelliWindow-symbol" data-definition-place="undefined-in-file">length</a>)
139eeb7e5b3SAdam Hornáček<a class="l" name="133" href="#133">133</a>    }
140eeb7e5b3SAdam Hornáček<a class="l" name="134" href="#134">134</a>    <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=zipWithIndex" class="intelliWindow-symbol" data-definition-place="undefined-in-file">zipWithIndex</a>.<a href="/source/s?defs=foreach" class="intelliWindow-symbol" data-definition-place="undefined-in-file">foreach</a> { <b>case</b> (<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>, <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) =&gt;
141eeb7e5b3SAdam Hornáček<a class="l" name="135" href="#135">135</a>      <b>if</b> (<a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>) {
142eeb7e5b3SAdam Hornáček<a class="l" name="136" href="#136">136</a>        <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a>(<a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) = <a href="/source/s?defs=calcChecksum" class="intelliWindow-symbol" data-definition-place="undefined-in-file">calcChecksum</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>)
143eeb7e5b3SAdam Hornáček<a class="l" name="137" href="#137">137</a>      }
144eeb7e5b3SAdam Hornáček<a class="l" name="138" href="#138">138</a>      <b>val</b> <a class="d" name="pieceId"/><a href="/source/s?refs=pieceId" class="d intelliWindow-symbol" data-definition-place="def">pieceId</a> = <a href="/source/s?defs=BroadcastBlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BroadcastBlockId</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <span class="s">&quot;piece&quot;</span> + <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>)
145eeb7e5b3SAdam Hornáček<a class="l" name="139" href="#139">139</a>      <b>val</b> <a class="d" name="bytes"/><a href="/source/s?refs=bytes" class="d intelliWindow-symbol" data-definition-place="def">bytes</a> = <b>new</b> <a href="/source/s?defs=ChunkedByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBuffer</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=duplicate" class="intelliWindow-symbol" data-definition-place="undefined-in-file">duplicate</a>())
146eeb7e5b3SAdam Hornáček<a class="hl" name="140" href="#140">140</a>      <b>if</b> (!<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=putBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>, <a href="/source/s?defs=bytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">bytes</a>, <a href="/source/s?defs=MEMORY_AND_DISK_SER" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK_SER</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>true</b>)) {
147eeb7e5b3SAdam Hornáček<a class="l" name="141" href="#141">141</a>        <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">&quot;Failed to store $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in local BlockManager&quot;</span>)
148eeb7e5b3SAdam Hornáček<a class="l" name="142" href="#142">142</a>      }
149eeb7e5b3SAdam Hornáček<a class="l" name="143" href="#143">143</a>    }
150eeb7e5b3SAdam Hornáček<a class="l" name="144" href="#144">144</a>    <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=length" class="intelliWindow-symbol" data-definition-place="undefined-in-file">length</a>
151eeb7e5b3SAdam Hornáček<a class="l" name="145" href="#145">145</a>  }
152eeb7e5b3SAdam Hornáček<a class="l" name="146" href="#146">146</a>
153eeb7e5b3SAdam Hornáček<a class="l" name="147" href="#147">147</a>  <span class="c">/** Fetch torrent blocks from the driver <a href="/source/s?path=and/">and</a>/<a href="/source/s?path=and/or">or</a> other executors. */</span>
154eeb7e5b3SAdam Hornáček<a class="l" name="148" href="#148">148</a>  <b>private</b> <b>def</b> <a href="/source/s?defs=readBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBlocks</a>(): <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=BlockData" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BlockData</a>] = {
155eeb7e5b3SAdam Hornáček<a class="l" name="149" href="#149">149</a>    <span class="c">// Fetch chunks of data. Note that all these chunks are stored in the BlockManager and reported</span>
156eeb7e5b3SAdam Hornáček<a class="hl" name="150" href="#150">150</a>    <span class="c">// to the driver, so other executors can pull these chunks from this executor as well.</span>
157eeb7e5b3SAdam Hornáček<a class="l" name="151" href="#151">151</a>    <b>val</b> <a class="d" name="blocks"/><a href="/source/s?refs=blocks" class="d intelliWindow-symbol" data-definition-place="def">blocks</a> = <b>new</b> <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=BlockData" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BlockData</a>](<a href="/source/s?defs=numBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">numBlocks</a>)
158eeb7e5b3SAdam Hornáček<a class="l" name="152" href="#152">152</a>    <b>val</b> <a class="d" name="bm"/><a href="/source/s?refs=bm" class="d intelliWindow-symbol" data-definition-place="def">bm</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>
159eeb7e5b3SAdam Hornáček<a class="l" name="153" href="#153">153</a>
160eeb7e5b3SAdam Hornáček<a class="l" name="154" href="#154">154</a>    <b>for</b> (<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a> &lt;- <a href="/source/s?defs=Random" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Random</a>.<a href="/source/s?defs=shuffle" class="intelliWindow-symbol" data-definition-place="undefined-in-file">shuffle</a>(<a href="/source/s?defs=Seq" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Seq</a>.<a href="/source/s?defs=range" class="intelliWindow-symbol" data-definition-place="undefined-in-file">range</a>(<span class="n">0</span>, <a href="/source/s?defs=numBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">numBlocks</a>))) {
161eeb7e5b3SAdam Hornáček<a class="l" name="155" href="#155">155</a>      <b>val</b> <a class="d" name="pieceId"/><a href="/source/s?refs=pieceId" class="d intelliWindow-symbol" data-definition-place="def">pieceId</a> = <a href="/source/s?defs=BroadcastBlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BroadcastBlockId</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <span class="s">&quot;piece&quot;</span> + <a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>)
162eeb7e5b3SAdam Hornáček<a class="l" name="156" href="#156">156</a>      <a href="/source/s?defs=logDebug" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logDebug</a>(s<span class="s">&quot;Reading piece $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">&quot;</span>)
163eeb7e5b3SAdam Hornáček<a class="l" name="157" href="#157">157</a>      <span class="c">// First try getLocalBytes because there is a chance that previous attempts to fetch the</span>
164eeb7e5b3SAdam Hornáček<a class="l" name="158" href="#158">158</a>      <span class="c">// broadcast blocks have already fetched some of the blocks. In that case, some blocks</span>
165eeb7e5b3SAdam Hornáček<a class="l" name="159" href="#159">159</a>      <span class="c">// would be available locally (on this executor).</span>
166eeb7e5b3SAdam Hornáček<a class="hl" name="160" href="#160">160</a>      <a class="d intelliWindow-symbol" href="#bm" data-definition-place="defined-in-file">bm</a>.<a href="/source/s?defs=getLocalBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getLocalBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>) <b>match</b> {
167eeb7e5b3SAdam Hornáček<a class="l" name="161" href="#161">161</a>        <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>) =&gt;
168eeb7e5b3SAdam Hornáček<a class="l" name="162" href="#162">162</a>          <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>(<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>) = <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>
169eeb7e5b3SAdam Hornáček<a class="l" name="163" href="#163">163</a>          <a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>)
170eeb7e5b3SAdam Hornáček<a class="l" name="164" href="#164">164</a>        <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> =&gt;
171eeb7e5b3SAdam Hornáček<a class="l" name="165" href="#165">165</a>          <a class="d intelliWindow-symbol" href="#bm" data-definition-place="defined-in-file">bm</a>.<a href="/source/s?defs=getRemoteBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getRemoteBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>) <b>match</b> {
172eeb7e5b3SAdam Hornáček<a class="l" name="166" href="#166">166</a>            <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>) =&gt;
173eeb7e5b3SAdam Hornáček<a class="l" name="167" href="#167">167</a>              <b>if</b> (<a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>) {
174eeb7e5b3SAdam Hornáček<a class="l" name="168" href="#168">168</a>                <b>val</b> <a class="d" name="sum"/><a href="/source/s?refs=sum" class="d intelliWindow-symbol" data-definition-place="def">sum</a> = <a href="/source/s?defs=calcChecksum" class="intelliWindow-symbol" data-definition-place="undefined-in-file">calcChecksum</a>(<a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>.<a href="/source/s?defs=chunks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">chunks</a>(<span class="n">0</span>))
175eeb7e5b3SAdam Hornáček<a class="l" name="169" href="#169">169</a>                <b>if</b> (<a class="d intelliWindow-symbol" href="#sum" data-definition-place="defined-in-file">sum</a> != <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a>(<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>)) {
176eeb7e5b3SAdam Hornáček<a class="hl" name="170" href="#170">170</a>                  <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">&quot;corrupt remote block $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">:&quot;</span> +
177eeb7e5b3SAdam Hornáček<a class="l" name="171" href="#171">171</a>                    s<span class="s">&quot; $</span><a class="d intelliWindow-symbol" href="#sum" data-definition-place="defined-in-file">sum</a><span class="s"> != ${checksums(pid)}&quot;</span>)
178eeb7e5b3SAdam Hornáček<a class="l" name="172" href="#172">172</a>                }
179eeb7e5b3SAdam Hornáček<a class="l" name="173" href="#173">173</a>              }
180eeb7e5b3SAdam Hornáček<a class="l" name="174" href="#174">174</a>              <span class="c">// We found the block from remote <a href="/source/s?path=executors/">executors</a>/<a href="/source/s?path=executors/driver">driver</a>&apos;s BlockManager, so put the block</span>
181eeb7e5b3SAdam Hornáček<a class="l" name="175" href="#175">175</a>              <span class="c">// in this executor&apos;s BlockManager.</span>
182eeb7e5b3SAdam Hornáček<a class="l" name="176" href="#176">176</a>              <b>if</b> (!<a class="d intelliWindow-symbol" href="#bm" data-definition-place="defined-in-file">bm</a>.<a href="/source/s?defs=putBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>, <a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>, <a href="/source/s?defs=StorageLevel" class="intelliWindow-symbol" data-definition-place="undefined-in-file">StorageLevel</a>.<a href="/source/s?defs=MEMORY_AND_DISK_SER" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK_SER</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>true</b>)) {
183eeb7e5b3SAdam Hornáček<a class="l" name="177" href="#177">177</a>                <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(
184eeb7e5b3SAdam Hornáček<a class="l" name="178" href="#178">178</a>                  s<span class="s">&quot;Failed to store $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in local BlockManager&quot;</span>)
185eeb7e5b3SAdam Hornáček<a class="l" name="179" href="#179">179</a>              }
186eeb7e5b3SAdam Hornáček<a class="hl" name="180" href="#180">180</a>              <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>(<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>) = <b>new</b> <a href="/source/s?defs=ByteBufferBlockData" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBufferBlockData</a>(<a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>, <b>true</b>)
187eeb7e5b3SAdam Hornáček<a class="l" name="181" href="#181">181</a>            <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> =&gt;
188eeb7e5b3SAdam Hornáček<a class="l" name="182" href="#182">182</a>              <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">&quot;Failed to get $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">&quot;</span>)
189eeb7e5b3SAdam Hornáček<a class="l" name="183" href="#183">183</a>          }
190eeb7e5b3SAdam Hornáček<a class="l" name="184" href="#184">184</a>      }
191eeb7e5b3SAdam Hornáček<a class="l" name="185" href="#185">185</a>    }
192eeb7e5b3SAdam Hornáček<a class="l" name="186" href="#186">186</a>    <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>
193eeb7e5b3SAdam Hornáček<a class="l" name="187" href="#187">187</a>  }
194eeb7e5b3SAdam Hornáček<a class="l" name="188" href="#188">188</a>
195eeb7e5b3SAdam Hornáček<a class="l" name="189" href="#189">189</a>  <span class="c">/**</span>
196eeb7e5b3SAdam Hornáček<a class="hl" name="190" href="#190">190</a><span class="c">   * Remove all persisted state associated with this Torrent broadcast on the executors.</span>
197eeb7e5b3SAdam Hornáček<a class="l" name="191" href="#191">191</a><span class="c">   */</span>
198eeb7e5b3SAdam Hornáček<a class="l" name="192" href="#192">192</a>  <b>override</b> <b>protected</b> <b>def</b> <a href="/source/s?defs=doUnpersist" class="intelliWindow-symbol" data-definition-place="undefined-in-file">doUnpersist</a>(<a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>) {
199eeb7e5b3SAdam Hornáček<a class="l" name="193" href="#193">193</a>    <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#unpersist" data-definition-place="defined-in-file">unpersist</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a> = <b>false</b>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>)
200eeb7e5b3SAdam Hornáček<a class="l" name="194" href="#194">194</a>  }
201eeb7e5b3SAdam Hornáček<a class="l" name="195" href="#195">195</a>
202eeb7e5b3SAdam Hornáček<a class="l" name="196" href="#196">196</a>  <span class="c">/**</span>
203eeb7e5b3SAdam Hornáček<a class="l" name="197" href="#197">197</a><span class="c">   * Remove all persisted state associated with this Torrent broadcast on the executors</span>
204eeb7e5b3SAdam Hornáček<a class="l" name="198" href="#198">198</a><span class="c">   * and driver.</span>
205eeb7e5b3SAdam Hornáček<a class="l" name="199" href="#199">199</a><span class="c">   */</span>
206eeb7e5b3SAdam Hornáček<a class="hl" name="200" href="#200">200</a>  <b>override</b> <b>protected</b> <b>def</b> <a href="/source/s?defs=doDestroy" class="intelliWindow-symbol" data-definition-place="undefined-in-file">doDestroy</a>(<a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>) {
207eeb7e5b3SAdam Hornáček<a class="l" name="201" href="#201">201</a>    <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#unpersist" data-definition-place="defined-in-file">unpersist</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a> = <b>true</b>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>)
208eeb7e5b3SAdam Hornáček<a class="l" name="202" href="#202">202</a>  }
209eeb7e5b3SAdam Hornáček<a class="l" name="203" href="#203">203</a>
210eeb7e5b3SAdam Hornáček<a class="l" name="204" href="#204">204</a>  <span class="c">/** Used by the JVM when serializing this object. */</span>
211eeb7e5b3SAdam Hornáček<a class="l" name="205" href="#205">205</a>  <b>private</b> <b>def</b> <a href="/source/s?defs=writeObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeObject</a>(<a class="d intelliWindow-symbol" href="#out" data-definition-place="defined-in-file">out</a>: <a href="/source/s?defs=ObjectOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ObjectOutputStream</a>): <a href="/source/s?defs=Unit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Unit</a> = <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryOrIOException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryOrIOException</a> {
212eeb7e5b3SAdam Hornáček<a class="l" name="206" href="#206">206</a>    <a href="/source/s?defs=assertValid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">assertValid</a>()
213eeb7e5b3SAdam Hornáček<a class="l" name="207" href="#207">207</a>    <a class="d intelliWindow-symbol" href="#out" data-definition-place="defined-in-file">out</a>.<a href="/source/s?defs=defaultWriteObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">defaultWriteObject</a>()
214eeb7e5b3SAdam Hornáček<a class="l" name="208" href="#208">208</a>  }
215eeb7e5b3SAdam Hornáček<a class="l" name="209" href="#209">209</a>
216eeb7e5b3SAdam Hornáček<a class="hl" name="210" href="#210">210</a>  <b>private</b> <b>def</b> <a href="/source/s?defs=readBroadcastBlock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBroadcastBlock</a>(): <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a> = <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryOrIOException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryOrIOException</a> {
217eeb7e5b3SAdam Hornáček<a class="l" name="211" href="#211">211</a>    <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a href="/source/s?defs=synchronized" class="intelliWindow-symbol" data-definition-place="undefined-in-file">synchronized</a> {
218eeb7e5b3SAdam Hornáček<a class="l" name="212" href="#212">212</a>      <a href="/source/s?defs=setConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">setConf</a>(<a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>)
219eeb7e5b3SAdam Hornáček<a class="l" name="213" href="#213">213</a>      <b>val</b> <a class="d" name="blockManager"/><a href="/source/s?refs=blockManager" class="d intelliWindow-symbol" data-definition-place="def">blockManager</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>
220eeb7e5b3SAdam Hornáček<a class="l" name="214" href="#214">214</a>      <a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=getLocalValues" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getLocalValues</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>) <b>match</b> {
221eeb7e5b3SAdam Hornáček<a class="l" name="215" href="#215">215</a>        <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=blockResult" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockResult</a>) =&gt;
222eeb7e5b3SAdam Hornáček<a class="l" name="216" href="#216">216</a>          <b>if</b> (<a href="/source/s?defs=blockResult" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockResult</a>.<a href="/source/s?defs=data" class="intelliWindow-symbol" data-definition-place="undefined-in-file">data</a>.<a href="/source/s?defs=hasNext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">hasNext</a>) {
223eeb7e5b3SAdam Hornáček<a class="l" name="217" href="#217">217</a>            <b>val</b> <a class="d" name="x"/><a href="/source/s?refs=x" class="d intelliWindow-symbol" data-definition-place="def">x</a> = <a href="/source/s?defs=blockResult" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockResult</a>.<a href="/source/s?defs=data" class="intelliWindow-symbol" data-definition-place="undefined-in-file">data</a>.<a href="/source/s?defs=next" class="intelliWindow-symbol" data-definition-place="undefined-in-file">next</a>().<a href="/source/s?defs=asInstanceOf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">asInstanceOf</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>]
224eeb7e5b3SAdam Hornáček<a class="l" name="218" href="#218">218</a>            <a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>)
225eeb7e5b3SAdam Hornáček<a class="l" name="219" href="#219">219</a>            <a class="d intelliWindow-symbol" href="#x" data-definition-place="defined-in-file">x</a>
226eeb7e5b3SAdam Hornáček<a class="hl" name="220" href="#220">220</a>          } <b>else</b> {
227eeb7e5b3SAdam Hornáček<a class="l" name="221" href="#221">221</a>            <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">&quot;Failed to get locally stored broadcast data: $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">&quot;</span>)
228eeb7e5b3SAdam Hornáček<a class="l" name="222" href="#222">222</a>          }
229eeb7e5b3SAdam Hornáček<a class="l" name="223" href="#223">223</a>        <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> =&gt;
230eeb7e5b3SAdam Hornáček<a class="l" name="224" href="#224">224</a>          <a href="/source/s?defs=logInfo" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logInfo</a>(<span class="s">&quot;Started reading broadcast variable &quot;</span> + <a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>)
231eeb7e5b3SAdam Hornáček<a class="l" name="225" href="#225">225</a>          <b>val</b> <a class="d" name="startTimeMs"/><a href="/source/s?refs=startTimeMs" class="d intelliWindow-symbol" data-definition-place="def">startTimeMs</a> = <a href="/source/s?defs=System" class="intelliWindow-symbol" data-definition-place="undefined-in-file">System</a>.<a href="/source/s?defs=currentTimeMillis" class="intelliWindow-symbol" data-definition-place="undefined-in-file">currentTimeMillis</a>()
232eeb7e5b3SAdam Hornáček<a class="l" name="226" href="#226">226</a>          <b>val</b> <a class="d" name="blocks"/><a href="/source/s?refs=blocks" class="d intelliWindow-symbol" data-definition-place="def">blocks</a> = <a href="/source/s?defs=readBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBlocks</a>()
233eeb7e5b3SAdam Hornáček<a class="l" name="227" href="#227">227</a>          <a href="/source/s?defs=logInfo" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logInfo</a>(<span class="s">&quot;Reading broadcast variable &quot;</span> + <a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a> + <span class="s">&quot; took&quot;</span> + <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=getUsedTimeMs" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getUsedTimeMs</a>(<a class="d intelliWindow-symbol" href="#startTimeMs" data-definition-place="defined-in-file">startTimeMs</a>))
234eeb7e5b3SAdam Hornáček<a class="l" name="228" href="#228">228</a>
235eeb7e5b3SAdam Hornáček<a class="l" name="229" href="#229">229</a>          <b>try</b> {
236eeb7e5b3SAdam Hornáček<a class="hl" name="230" href="#230">230</a>            <b>val</b> <a class="d" name="obj"/><a href="/source/s?refs=obj" class="d intelliWindow-symbol" data-definition-place="def">obj</a> = <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#unBlockifyObject" data-definition-place="defined-in-file">unBlockifyObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>](
237eeb7e5b3SAdam Hornáček<a class="l" name="231" href="#231">231</a>              <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=map" class="intelliWindow-symbol" data-definition-place="undefined-in-file">map</a>(<b>_</b>.<a href="/source/s?defs=toInputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toInputStream</a>()), <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>, <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>)
238eeb7e5b3SAdam Hornáček<a class="l" name="232" href="#232">232</a>            <span class="c">// Store the merged copy in BlockManager so other tasks on this executor don&apos;t</span>
239eeb7e5b3SAdam Hornáček<a class="l" name="233" href="#233">233</a>            <span class="c">// need to re-fetch it.</span>
240eeb7e5b3SAdam Hornáček<a class="l" name="234" href="#234">234</a>            <b>val</b> <a class="d" name="storageLevel"/><a href="/source/s?refs=storageLevel" class="d intelliWindow-symbol" data-definition-place="def">storageLevel</a> = <a href="/source/s?defs=StorageLevel" class="intelliWindow-symbol" data-definition-place="undefined-in-file">StorageLevel</a>.<a href="/source/s?defs=MEMORY_AND_DISK" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK</a>
241eeb7e5b3SAdam Hornáček<a class="l" name="235" href="#235">235</a>            <b>if</b> (!<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=putSingle" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putSingle</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>, <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>, <a class="d intelliWindow-symbol" href="#storageLevel" data-definition-place="defined-in-file">storageLevel</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>false</b>)) {
242eeb7e5b3SAdam Hornáček<a class="l" name="236" href="#236">236</a>              <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">&quot;Failed to store $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in BlockManager&quot;</span>)
243eeb7e5b3SAdam Hornáček<a class="l" name="237" href="#237">237</a>            }
244eeb7e5b3SAdam Hornáček<a class="l" name="238" href="#238">238</a>            <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>
245eeb7e5b3SAdam Hornáček<a class="l" name="239" href="#239">239</a>          } <b>finally</b> {
246eeb7e5b3SAdam Hornáček<a class="hl" name="240" href="#240">240</a>            <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=foreach" class="intelliWindow-symbol" data-definition-place="undefined-in-file">foreach</a>(<b>_</b>.<a href="/source/s?defs=dispose" class="intelliWindow-symbol" data-definition-place="undefined-in-file">dispose</a>())
247eeb7e5b3SAdam Hornáček<a class="l" name="241" href="#241">241</a>          }
248eeb7e5b3SAdam Hornáček<a class="l" name="242" href="#242">242</a>      }
249eeb7e5b3SAdam Hornáček<a class="l" name="243" href="#243">243</a>    }
250eeb7e5b3SAdam Hornáček<a class="l" name="244" href="#244">244</a>  }
251eeb7e5b3SAdam Hornáček<a class="l" name="245" href="#245">245</a>
252eeb7e5b3SAdam Hornáček<a class="l" name="246" href="#246">246</a>  <span class="c">/**</span>
253eeb7e5b3SAdam Hornáček<a class="l" name="247" href="#247">247</a><span class="c">   * If running in a task, register the given block&apos;s locks for release upon task completion.</span>
254eeb7e5b3SAdam Hornáček<a class="l" name="248" href="#248">248</a><span class="c">   * Otherwise, if not running in a task then immediately release the lock.</span>
255eeb7e5b3SAdam Hornáček<a class="l" name="249" href="#249">249</a><span class="c">   */</span>
256eeb7e5b3SAdam Hornáček<a class="hl" name="250" href="#250">250</a>  <b>private</b> <b>def</b> <a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=blockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockId</a>: <a href="/source/s?defs=BlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BlockId</a>): <a href="/source/s?defs=Unit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Unit</a> = {
257eeb7e5b3SAdam Hornáček<a class="l" name="251" href="#251">251</a>    <b>val</b> <a class="d" name="blockManager"/><a href="/source/s?refs=blockManager" class="d intelliWindow-symbol" data-definition-place="def">blockManager</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>
258eeb7e5b3SAdam Hornáček<a class="l" name="252" href="#252">252</a>    <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>(<a href="/source/s?defs=TaskContext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">TaskContext</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>()) <b>match</b> {
259eeb7e5b3SAdam Hornáček<a class="l" name="253" href="#253">253</a>      <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=taskContext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">taskContext</a>) =&gt;
260eeb7e5b3SAdam Hornáček<a class="l" name="254" href="#254">254</a>        <a href="/source/s?defs=taskContext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">taskContext</a>.<a href="/source/s?defs=addTaskCompletionListener" class="intelliWindow-symbol" data-definition-place="undefined-in-file">addTaskCompletionListener</a>(<b>_</b> =&gt; <a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=blockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockId</a>))
261eeb7e5b3SAdam Hornáček<a class="l" name="255" href="#255">255</a>      <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> =&gt;
262eeb7e5b3SAdam Hornáček<a class="l" name="256" href="#256">256</a>        <span class="c">// This should only happen on the driver, where broadcast variables may be accessed</span>
263eeb7e5b3SAdam Hornáček<a class="l" name="257" href="#257">257</a>        <span class="c">// outside of running tasks (e.g. when computing rdd.partitions()). In order to allow</span>
264eeb7e5b3SAdam Hornáček<a class="l" name="258" href="#258">258</a>        <span class="c">// broadcast variables to be garbage collected we need to free the reference here</span>
265eeb7e5b3SAdam Hornáček<a class="l" name="259" href="#259">259</a>        <span class="c">// which is slightly unsafe but is technically okay because broadcast variables aren&apos;t</span>
266eeb7e5b3SAdam Hornáček<a class="hl" name="260" href="#260">260</a>        <span class="c">// stored off-heap.</span>
267eeb7e5b3SAdam Hornáček<a class="l" name="261" href="#261">261</a>        <a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=blockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockId</a>)
268eeb7e5b3SAdam Hornáček<a class="l" name="262" href="#262">262</a>    }
269eeb7e5b3SAdam Hornáček<a class="l" name="263" href="#263">263</a>  }
270eeb7e5b3SAdam Hornáček<a class="l" name="264" href="#264">264</a>
271eeb7e5b3SAdam Hornáček<a class="l" name="265" href="#265">265</a>}
272eeb7e5b3SAdam Hornáček<a class="l" name="266" href="#266">266</a>
273eeb7e5b3SAdam Hornáček<a class="l" name="267" href="#267">267</a>
274eeb7e5b3SAdam Hornáček<a class="l" name="268" href="#268">268</a><b>private</b> <b>object</b> <a class="d" name="TorrentBroadcast"/><a href="/source/s?refs=TorrentBroadcast" class="d intelliWindow-symbol" data-definition-place="def">TorrentBroadcast</a> <b>extends</b> <a href="/source/s?defs=Logging" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Logging</a> {
275eeb7e5b3SAdam Hornáček<a class="l" name="269" href="#269">269</a>
276eeb7e5b3SAdam Hornáček<a class="hl" name="270" href="#270">270</a>  <b>def</b> <a class="xmt" name="blockifyObject"/><a href="/source/s?refs=blockifyObject" class="xmt intelliWindow-symbol" data-definition-place="def">blockifyObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>: <a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a>](
277eeb7e5b3SAdam Hornáček<a class="l" name="271" href="#271">271</a>      <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>,
278eeb7e5b3SAdam Hornáček<a class="l" name="272" href="#272">272</a>      <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>: <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a>,
279eeb7e5b3SAdam Hornáček<a class="l" name="273" href="#273">273</a>      <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>: <a href="/source/s?defs=Serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializer</a>,
280eeb7e5b3SAdam Hornáček<a class="l" name="274" href="#274">274</a>      <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>: <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>[<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>]): <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a>] = {
281eeb7e5b3SAdam Hornáček<a class="l" name="275" href="#275">275</a>    <b>val</b> <a class="d" name="cbbos"/><a href="/source/s?refs=cbbos" class="d intelliWindow-symbol" data-definition-place="def">cbbos</a> = <b>new</b> <a href="/source/s?defs=ChunkedByteBufferOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBufferOutputStream</a>(<a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>, <a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a>.<a href="/source/s?defs=allocate" class="intelliWindow-symbol" data-definition-place="undefined-in-file">allocate</a>)
282eeb7e5b3SAdam Hornáček<a class="l" name="276" href="#276">276</a>    <b>val</b> <a class="d" name="out"/><a href="/source/s?refs=out" class="d intelliWindow-symbol" data-definition-place="def">out</a> = <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>.<a href="/source/s?defs=map" class="intelliWindow-symbol" data-definition-place="undefined-in-file">map</a>(<a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a> =&gt; <a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a>.<a href="/source/s?defs=compressedOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressedOutputStream</a>(<a class="d intelliWindow-symbol" href="#cbbos" data-definition-place="defined-in-file">cbbos</a>)).<a href="/source/s?defs=getOrElse" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getOrElse</a>(<a class="d intelliWindow-symbol" href="#cbbos" data-definition-place="defined-in-file">cbbos</a>)
283eeb7e5b3SAdam Hornáček<a class="l" name="277" href="#277">277</a>    <b>val</b> <a class="d" name="ser"/><a href="/source/s?refs=ser" class="d intelliWindow-symbol" data-definition-place="def">ser</a> = <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>.<a href="/source/s?defs=newInstance" class="intelliWindow-symbol" data-definition-place="undefined-in-file">newInstance</a>()
284eeb7e5b3SAdam Hornáček<a class="l" name="278" href="#278">278</a>    <b>val</b> <a class="d" name="serOut"/><a href="/source/s?refs=serOut" class="d intelliWindow-symbol" data-definition-place="def">serOut</a> = <a href="/source/s?defs=ser" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ser</a>.<a href="/source/s?defs=serializeStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializeStream</a>(<a class="d intelliWindow-symbol" href="#out" data-definition-place="defined-in-file">out</a>)
285eeb7e5b3SAdam Hornáček<a class="l" name="279" href="#279">279</a>    <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryWithSafeFinally" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryWithSafeFinally</a> {
286eeb7e5b3SAdam Hornáček<a class="hl" name="280" href="#280">280</a>      <a class="d intelliWindow-symbol" href="#serOut" data-definition-place="defined-in-file">serOut</a>.<a href="/source/s?defs=writeObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>](<a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>)
287eeb7e5b3SAdam Hornáček<a class="l" name="281" href="#281">281</a>    } {
288eeb7e5b3SAdam Hornáček<a class="l" name="282" href="#282">282</a>      <a class="d intelliWindow-symbol" href="#serOut" data-definition-place="defined-in-file">serOut</a>.<a href="/source/s?defs=close" class="intelliWindow-symbol" data-definition-place="undefined-in-file">close</a>()
289eeb7e5b3SAdam Hornáček<a class="l" name="283" href="#283">283</a>    }
290eeb7e5b3SAdam Hornáček<a class="l" name="284" href="#284">284</a>    <a class="d intelliWindow-symbol" href="#cbbos" data-definition-place="defined-in-file">cbbos</a>.<a href="/source/s?defs=toChunkedByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toChunkedByteBuffer</a>.<a href="/source/s?defs=getChunks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getChunks</a>()
291eeb7e5b3SAdam Hornáček<a class="l" name="285" href="#285">285</a>  }
292eeb7e5b3SAdam Hornáček<a class="l" name="286" href="#286">286</a>
293eeb7e5b3SAdam Hornáček<a class="l" name="287" href="#287">287</a>  <b>def</b> <a class="xmt" name="unBlockifyObject"/><a href="/source/s?refs=unBlockifyObject" class="xmt intelliWindow-symbol" data-definition-place="def">unBlockifyObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>: <a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a>](
294eeb7e5b3SAdam Hornáček<a class="l" name="288" href="#288">288</a>      <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>: <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=InputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">InputStream</a>],
295eeb7e5b3SAdam Hornáček<a class="l" name="289" href="#289">289</a>      <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>: <a href="/source/s?defs=Serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializer</a>,
296eeb7e5b3SAdam Hornáček<a class="hl" name="290" href="#290">290</a>      <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>: <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>[<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>]): <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a> = {
297eeb7e5b3SAdam Hornáček<a class="l" name="291" href="#291">291</a>    <a href="/source/s?defs=require" class="intelliWindow-symbol" data-definition-place="undefined-in-file">require</a>(<a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=nonEmpty" class="intelliWindow-symbol" data-definition-place="undefined-in-file">nonEmpty</a>, <span class="s">&quot;Cannot unblockify an empty array of blocks&quot;</span>)
298eeb7e5b3SAdam Hornáček<a class="l" name="292" href="#292">292</a>    <b>val</b> <a class="d" name="is"/><a href="/source/s?refs=is" class="d intelliWindow-symbol" data-definition-place="def">is</a> = <b>new</b> <a href="/source/s?defs=SequenceInputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SequenceInputStream</a>(<a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=iterator" class="intelliWindow-symbol" data-definition-place="undefined-in-file">iterator</a>.<a href="/source/s?defs=asJavaEnumeration" class="intelliWindow-symbol" data-definition-place="undefined-in-file">asJavaEnumeration</a>)
299eeb7e5b3SAdam Hornáček<a class="l" name="293" href="#293">293</a>    <b>val</b> <a class="d" name="in"/><a href="/source/s?refs=in" class="d intelliWindow-symbol" data-definition-place="def">in</a>: <a href="/source/s?defs=InputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">InputStream</a> = <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>.<a href="/source/s?defs=map" class="intelliWindow-symbol" data-definition-place="undefined-in-file">map</a>(<a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a> =&gt; <a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a>.<a href="/source/s?defs=compressedInputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressedInputStream</a>(<a class="d intelliWindow-symbol" href="#is" data-definition-place="defined-in-file">is</a>)).<a href="/source/s?defs=getOrElse" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getOrElse</a>(<a class="d intelliWindow-symbol" href="#is" data-definition-place="defined-in-file">is</a>)
300eeb7e5b3SAdam Hornáček<a class="l" name="294" href="#294">294</a>    <b>val</b> <a class="d" name="ser"/><a href="/source/s?refs=ser" class="d intelliWindow-symbol" data-definition-place="def">ser</a> = <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>.<a href="/source/s?defs=newInstance" class="intelliWindow-symbol" data-definition-place="undefined-in-file">newInstance</a>()
301eeb7e5b3SAdam Hornáček<a class="l" name="295" href="#295">295</a>    <b>val</b> <a class="d" name="serIn"/><a href="/source/s?refs=serIn" class="d intelliWindow-symbol" data-definition-place="def">serIn</a> = <a href="/source/s?defs=ser" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ser</a>.<a href="/source/s?defs=deserializeStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">deserializeStream</a>(<a class="d intelliWindow-symbol" href="#in" data-definition-place="defined-in-file">in</a>)
302eeb7e5b3SAdam Hornáček<a class="l" name="296" href="#296">296</a>    <b>val</b> <a class="d" name="obj"/><a href="/source/s?refs=obj" class="d intelliWindow-symbol" data-definition-place="def">obj</a> = <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryWithSafeFinally" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryWithSafeFinally</a> {
303eeb7e5b3SAdam Hornáček<a class="l" name="297" href="#297">297</a>      <a class="d intelliWindow-symbol" href="#serIn" data-definition-place="defined-in-file">serIn</a>.<a href="/source/s?defs=readObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>]()
304eeb7e5b3SAdam Hornáček<a class="l" name="298" href="#298">298</a>    } {
305eeb7e5b3SAdam Hornáček<a class="l" name="299" href="#299">299</a>      <a class="d intelliWindow-symbol" href="#serIn" data-definition-place="defined-in-file">serIn</a>.`<a href="/source/s?defs=close" class="intelliWindow-symbol" data-definition-place="undefined-in-file">close</a>`()
306eeb7e5b3SAdam Hornáček<a class="hl" name="300" href="#300">300</a>    }
307eeb7e5b3SAdam Hornáček<a class="l" name="301" href="#301">301</a>    <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>
308eeb7e5b3SAdam Hornáček<a class="l" name="302" href="#302">302</a>  }
309eeb7e5b3SAdam Hornáček<a class="l" name="303" href="#303">303</a>
310eeb7e5b3SAdam Hornáček<a class="l" name="304" href="#304">304</a>  <span class="c">/**</span>
311eeb7e5b3SAdam Hornáček<a class="l" name="305" href="#305">305</a><span class="c">   * Remove all persisted blocks associated with this torrent broadcast on the executors.</span>
312eeb7e5b3SAdam Hornáček<a class="l" name="306" href="#306">306</a><span class="c">   * If removeFromDriver is true, also remove these persisted blocks on the driver.</span>
313eeb7e5b3SAdam Hornáček<a class="l" name="307" href="#307">307</a><span class="c">   */</span>
314eeb7e5b3SAdam Hornáček<a class="l" name="308" href="#308">308</a>  <b>def</b> <a class="xmt" name="unpersist"/><a href="/source/s?refs=unpersist" class="xmt intelliWindow-symbol" data-definition-place="def">unpersist</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>: <a href="/source/s?defs=Long" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Long</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>): <a href="/source/s?defs=Unit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Unit</a> = {
315eeb7e5b3SAdam Hornáček<a class="l" name="309" href="#309">309</a>    <a href="/source/s?defs=logDebug" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logDebug</a>(s<span class="s">&quot;<a href="http://example.com?">http://example.com?</a>$</span><a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a><span class="s">&quot;</span>)
316eeb7e5b3SAdam Hornáček<a class="hl" name="310" href="#310">310</a>    <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=master" class="intelliWindow-symbol" data-definition-place="undefined-in-file">master</a>.<a href="/source/s?defs=removeBroadcast" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeBroadcast</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>)
317eeb7e5b3SAdam Hornáček<a class="l" name="311" href="#311">311</a>  }
318eeb7e5b3SAdam Hornáček<a class="l" name="312" href="#312">312</a>
319eeb7e5b3SAdam Hornáček<a class="l" name="313" href="#313">313</a>  <span class="c">/*<a href="http://example.com">http://example.com</a>.*/</span>
320eeb7e5b3SAdam Hornáček<a class="l" name="314" href="#314">314</a>  <span class="c">/* comment /* comment */</span>
321eeb7e5b3SAdam Hornáček<a class="l" name="315" href="#315">315</a><span class="c">  comment</span>
322eeb7e5b3SAdam Hornáček<a class="l" name="316" href="#316">316</a><span class="c">  */</span>
323eeb7e5b3SAdam Hornáček<a class="l" name="317" href="#317">317</a>
324eeb7e5b3SAdam Hornáček<a class="l" name="318" href="#318">318</a>  <b>def</b> <a class="xmt" name="unary_"/><a href="/source/s?refs=unary_" class="xmt intelliWindow-symbol" data-definition-place="def">unary_</a>~ = <span class="n">0</span>
325eeb7e5b3SAdam Hornáček<a class="l" name="319" href="#319">319</a>  <span class="c">/***/</span>
326eeb7e5b3SAdam Hornáček<a class="hl" name="320" href="#320">320</a>}
327eeb7e5b3SAdam Hornáček<a class="l" name="321" href="#321">321</a></body>
328eeb7e5b3SAdam Hornáček</html>
329